ClusterCockpit/cc-metric-collector
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-metric-collector.git synced 2025-04-16 09:55:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

Formatting

Browse Source
This commit is contained in:
Thomas Roehl 2021-12-21 14:04:31 +01:00
parent 988cea381e
commit a6feb16ec1
30 changed files with 775 additions and 793 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -16,5 +16,6 @@ fmt:
go fmt sinks/*.go go fmt sinks/*.go
go fmt receivers/*.go go fmt receivers/*.go
go fmt metric-collector.go go fmt metric-collector.go
find . -name "*.go" -exec go fmt {} \;
.PHONY: clean .PHONY: clean

181
collectors/collectorManager.go
View File

@ -1,16 +1,15 @@
package collectors package collectors
import ( import (
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" "encoding/json"
"sync" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"time" mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker"
"log" "log"
"os" "os"
"encoding/json" "sync"
mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker" "time"
) )
var AvailableCollectors = map[string]MetricCollector{ var AvailableCollectors = map[string]MetricCollector{
"likwid": &LikwidCollector{}, "likwid": &LikwidCollector{},
"loadavg": &LoadavgCollector{}, "loadavg": &LoadavgCollector{},
@ -27,15 +26,14 @@ var AvailableCollectors = map[string]MetricCollector{
"ipmistat": &IpmiCollector{}, "ipmistat": &IpmiCollector{},
} }
type collectorManager struct { type collectorManager struct {
collectors []MetricCollector collectors []MetricCollector
output chan lp.CCMetric output chan lp.CCMetric
done chan bool done chan bool
ticker mct.MultiChanTicker ticker mct.MultiChanTicker
duration time.Duration duration time.Duration
wg *sync.WaitGroup wg *sync.WaitGroup
config map[string]json.RawMessage config map[string]json.RawMessage
} }
type CollectorManager interface { type CollectorManager interface {
@ -45,96 +43,95 @@ type CollectorManager interface {
Close() Close()
} }
func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) error { func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) error {
cm.collectors = make([]MetricCollector, 0) cm.collectors = make([]MetricCollector, 0)
cm.output = nil cm.output = nil
cm.done = make(chan bool) cm.done = make(chan bool)
cm.wg = wg cm.wg = wg
cm.ticker = ticker cm.ticker = ticker
cm.duration = duration cm.duration = duration
configFile, err := os.Open(collectConfigFile) configFile, err := os.Open(collectConfigFile)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return err return err
} }
defer configFile.Close() defer configFile.Close()
jsonParser := json.NewDecoder(configFile) jsonParser := json.NewDecoder(configFile)
err = jsonParser.Decode(&cm.config) err = jsonParser.Decode(&cm.config)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return err return err
} }
for k, cfg := range cm.config { for k, cfg := range cm.config {
log.Print(k, " ", cfg) log.Print(k, " ", cfg)
if _, found := AvailableCollectors[k]; !found { if _, found := AvailableCollectors[k]; !found {
log.Print("[CollectorManager] SKIP unknown collector ", k) log.Print("[CollectorManager] SKIP unknown collector ", k)
continue continue
} }
c := AvailableCollectors[k] c := AvailableCollectors[k]
err = c.Init(cfg) err = c.Init(cfg)
if err != nil { if err != nil {
log.Print("[CollectorManager] Collector ", k, "initialization failed: ", err.Error()) log.Print("[CollectorManager] Collector ", k, "initialization failed: ", err.Error())
continue continue
} }
cm.collectors = append(cm.collectors, c) cm.collectors = append(cm.collectors, c)
} }
return nil return nil
} }
func (cm *collectorManager) Start() { func (cm *collectorManager) Start() {
cm.wg.Add(1) cm.wg.Add(1)
tick := make(chan time.Time) tick := make(chan time.Time)
cm.ticker.AddChannel(tick) cm.ticker.AddChannel(tick)
go func() { go func() {
for { for {
CollectorManagerLoop: CollectorManagerLoop:
select { select {
case <- cm.done: case <-cm.done:
for _, c := range cm.collectors { for _, c := range cm.collectors {
c.Close() c.Close()
} }
cm.wg.Done() cm.wg.Done()
log.Print("[CollectorManager] DONE\n") log.Print("[CollectorManager] DONE\n")
break CollectorManagerLoop break CollectorManagerLoop
case t := <- tick: case t := <-tick:
for _, c := range cm.collectors { for _, c := range cm.collectors {
CollectorManagerInputLoop: CollectorManagerInputLoop:
select { select {
case <- cm.done: case <-cm.done:
for _, c := range cm.collectors { for _, c := range cm.collectors {
c.Close() c.Close()
} }
cm.wg.Done() cm.wg.Done()
log.Print("[CollectorManager] DONE\n") log.Print("[CollectorManager] DONE\n")
break CollectorManagerInputLoop break CollectorManagerInputLoop
default: default:
log.Print("[CollectorManager] ", c.Name(), " ", t) log.Print("[CollectorManager] ", c.Name(), " ", t)
c.Read(cm.duration, cm.output) c.Read(cm.duration, cm.output)
} }
} }
} }
} }
log.Print("[CollectorManager] EXIT\n") log.Print("[CollectorManager] EXIT\n")
}() }()
log.Print("[CollectorManager] STARTED\n") log.Print("[CollectorManager] STARTED\n")
} }
func (cm *collectorManager) AddOutput(output chan lp.CCMetric) { func (cm *collectorManager) AddOutput(output chan lp.CCMetric) {
cm.output = output cm.output = output
} }
func (cm *collectorManager) Close() { func (cm *collectorManager) Close() {
cm.done <- true cm.done <- true
log.Print("[CollectorManager] CLOSE") log.Print("[CollectorManager] CLOSE")
} }
func New(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) (CollectorManager, error) { func New(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) (CollectorManager, error) {
cm := &collectorManager{} cm := &collectorManager{}
err := cm.Init(ticker, duration, wg, collectConfigFile) err := cm.Init(ticker, duration, wg, collectConfigFile)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return cm, err return cm, err
} }

2
collectors/cpustatMetric.go
View File

@ -24,7 +24,7 @@ type CpustatCollector struct {
func (m *CpustatCollector) Init(config json.RawMessage) error { func (m *CpustatCollector) Init(config json.RawMessage) error {
m.name = "CpustatCollector" m.name = "CpustatCollector"
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "CPU"} m.meta = map[string]string{"source": m.name, "group": "CPU"}
if len(config) > 0 { if len(config) > 0 {
err := json.Unmarshal(config, &m.config) err := json.Unmarshal(config, &m.config)
if err != nil { if err != nil {

2
collectors/customCmdMetric.go
View File

@ -32,7 +32,7 @@ type CustomCmdCollector struct {
func (m *CustomCmdCollector) Init(config json.RawMessage) error { func (m *CustomCmdCollector) Init(config json.RawMessage) error {
var err error var err error
m.name = "CustomCmdCollector" m.name = "CustomCmdCollector"
m.meta = map[string]string{"source" : m.name, "group" : "Custom"} m.meta = map[string]string{"source": m.name, "group": "Custom"}
if len(config) > 0 { if len(config) > 0 {
err = json.Unmarshal(config, &m.config) err = json.Unmarshal(config, &m.config)
if err != nil { if err != nil {

2
collectors/diskstatMetric.go
View File

@ -27,7 +27,7 @@ type DiskstatCollector struct {
func (m *DiskstatCollector) Init(config json.RawMessage) error { func (m *DiskstatCollector) Init(config json.RawMessage) error {
var err error var err error
m.name = "DiskstatCollector" m.name = "DiskstatCollector"
m.meta = map[string]string{"source" : m.name, "group" : "Disk"} m.meta = map[string]string{"source": m.name, "group": "Disk"}
m.setup() m.setup()
if len(config) > 0 { if len(config) > 0 {
err = json.Unmarshal(config, &m.config) err = json.Unmarshal(config, &m.config)

2
collectors/infinibandMetric.go
View File

@ -54,7 +54,7 @@ func (m *InfinibandCollector) Init(config json.RawMessage) error {
m.name = "InfinibandCollector" m.name = "InfinibandCollector"
m.use_perfquery = false m.use_perfquery = false
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "Network"} m.meta = map[string]string{"source": m.name, "group": "Network"}
m.tags = map[string]string{"type": "node"} m.tags = map[string]string{"type": "node"}
if len(config) > 0 { if len(config) > 0 {
err = json.Unmarshal(config, &m.config) err = json.Unmarshal(config, &m.config)

10
collectors/ipmiMetric.go
View File

@ -31,7 +31,7 @@ type IpmiCollector struct {
func (m *IpmiCollector) Init(config json.RawMessage) error { func (m *IpmiCollector) Init(config json.RawMessage) error {
m.name = "IpmiCollector" m.name = "IpmiCollector"
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "IPMI"} m.meta = map[string]string{"source": m.name, "group": "IPMI"}
if len(config) > 0 { if len(config) > 0 {
err := json.Unmarshal(config, &m.config) err := json.Unmarshal(config, &m.config)
if err != nil { if err != nil {
@ -85,7 +85,7 @@ func (m *IpmiCollector) readIpmiTool(cmd string, output chan lp.CCMetric) {
y, err := lp.New(name, map[string]string{"type": "node"}, m.meta, map[string]interface{}{"value": v}, time.Now()) y, err := lp.New(name, map[string]string{"type": "node"}, m.meta, map[string]interface{}{"value": v}, time.Now())
if err == nil { if err == nil {
y.AddMeta("unit", unit) y.AddMeta("unit", unit)
output <- y output <- y
} }
} }
@ -112,9 +112,9 @@ func (m *IpmiCollector) readIpmiSensors(cmd string, output chan lp.CCMetric) {
name := strings.ToLower(strings.Replace(lv[1], " ", "_", -1)) name := strings.ToLower(strings.Replace(lv[1], " ", "_", -1))
y, err := lp.New(name, map[string]string{"type": "node"}, m.meta, map[string]interface{}{"value": v}, time.Now()) y, err := lp.New(name, map[string]string{"type": "node"}, m.meta, map[string]interface{}{"value": v}, time.Now())
if err == nil { if err == nil {
if len(lv) > 4 { if len(lv) > 4 {
y.AddMeta("unit", lv[4]) y.AddMeta("unit", lv[4])
} }
output <- y output <- y
} }
} }

30
collectors/likwidMetric.go
View File

@ -27,21 +27,21 @@ import (
type MetricScope int type MetricScope int
const ( const (
METRIC_SCOPE_HWTHREAD = iota METRIC_SCOPE_HWTHREAD = iota
METRIC_SCOPE_SOCKET METRIC_SCOPE_SOCKET
METRIC_SCOPE_NUMA METRIC_SCOPE_NUMA
METRIC_SCOPE_NODE METRIC_SCOPE_NODE
) )
func (ms MetricScope) String() string { func (ms MetricScope) String() string {
return []string{"Head", "Shoulder", "Knee", "Toe"}[ms] return []string{"Head", "Shoulder", "Knee", "Toe"}[ms]
} }
type LikwidCollectorMetricConfig struct { type LikwidCollectorMetricConfig struct {
Name string `json:"name"` Name string `json:"name"`
Calc string `json:"calc"` Calc string `json:"calc"`
Scope MetricScope `json:"socket_scope"` Scope MetricScope `json:"socket_scope"`
Publish bool `json:"publish"` Publish bool `json:"publish"`
} }
type LikwidCollectorEventsetConfig struct { type LikwidCollectorEventsetConfig struct {
@ -127,13 +127,13 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
} }
} }
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "PerfCounter"} m.meta = map[string]string{"source": m.name, "group": "PerfCounter"}
cpulist := CpuList() cpulist := CpuList()
m.cpulist = make([]C.int, len(cpulist)) m.cpulist = make([]C.int, len(cpulist))
slist := getSocketCpus() slist := getSocketCpus()
m.sock2tid = make(map[int]int) m.sock2tid = make(map[int]int)
// m.numa2tid = make(map[int]int) // m.numa2tid = make(map[int]int)
for i, c := range cpulist { for i, c := range cpulist {
m.cpulist[i] = C.int(c) m.cpulist[i] = C.int(c)
if sid, found := slist[m.cpulist[i]]; found { if sid, found := slist[m.cpulist[i]]; found {
@ -264,7 +264,7 @@ func (m *LikwidCollector) Read(interval time.Duration, output chan lp.CCMetric)
for sid, tid := range m.sock2tid { for sid, tid := range m.sock2tid {
y, err := lp.New(metric.Name, y, err := lp.New(metric.Name,
map[string]string{"type": "socket", map[string]string{"type": "socket",
"type-id": fmt.Sprintf("%d", int(sid))}, "type-id": fmt.Sprintf("%d", int(sid))},
m.meta, m.meta,
map[string]interface{}{"value": m.mresults[i][tid][metric.Name]}, map[string]interface{}{"value": m.mresults[i][tid][metric.Name]},
time.Now()) time.Now())
@ -276,7 +276,7 @@ func (m *LikwidCollector) Read(interval time.Duration, output chan lp.CCMetric)
for tid, cpu := range m.cpulist { for tid, cpu := range m.cpulist {
y, err := lp.New(metric.Name, y, err := lp.New(metric.Name,
map[string]string{"type": "cpu", map[string]string{"type": "cpu",
"type-id": fmt.Sprintf("%d", int(cpu))}, "type-id": fmt.Sprintf("%d", int(cpu))},
m.meta, m.meta,
map[string]interface{}{"value": m.mresults[i][tid][metric.Name]}, map[string]interface{}{"value": m.mresults[i][tid][metric.Name]},
time.Now()) time.Now())
@ -295,7 +295,7 @@ func (m *LikwidCollector) Read(interval time.Duration, output chan lp.CCMetric)
for sid, tid := range m.sock2tid { for sid, tid := range m.sock2tid {
y, err := lp.New(metric.Name, y, err := lp.New(metric.Name,
map[string]string{"type": "socket", map[string]string{"type": "socket",
"type-id": fmt.Sprintf("%d", int(sid))}, "type-id": fmt.Sprintf("%d", int(sid))},
m.meta, m.meta,
map[string]interface{}{"value": m.gmresults[tid][metric.Name]}, map[string]interface{}{"value": m.gmresults[tid][metric.Name]},
time.Now()) time.Now())
@ -307,7 +307,7 @@ func (m *LikwidCollector) Read(interval time.Duration, output chan lp.CCMetric)
for tid, cpu := range m.cpulist { for tid, cpu := range m.cpulist {
y, err := lp.New(metric.Name, y, err := lp.New(metric.Name,
map[string]string{"type": "cpu", map[string]string{"type": "cpu",
"type-id": fmt.Sprintf("%d", int(cpu))}, "type-id": fmt.Sprintf("%d", int(cpu))},
m.meta, m.meta,
map[string]interface{}{"value": m.gmresults[tid][metric.Name]}, map[string]interface{}{"value": m.gmresults[tid][metric.Name]},
time.Now()) time.Now())

2
collectors/loadavgMetric.go
View File

@ -32,7 +32,7 @@ func (m *LoadavgCollector) Init(config json.RawMessage) error {
return err return err
} }
} }
m.meta = map[string]string{"source" : m.name, "group" : "LOAD"} m.meta = map[string]string{"source": m.name, "group": "LOAD"}
m.tags = map[string]string{"type": "node"} m.tags = map[string]string{"type": "node"}
m.load_matches = []string{"load_one", "load_five", "load_fifteen"} m.load_matches = []string{"load_one", "load_five", "load_fifteen"}
m.proc_matches = []string{"proc_run", "proc_total"} m.proc_matches = []string{"proc_run", "proc_total"}

8
collectors/lustreMetric.go
View File

@ -37,7 +37,7 @@ func (m *LustreCollector) Init(config json.RawMessage) error {
} }
m.setup() m.setup()
m.tags = map[string]string{"type": "node"} m.tags = map[string]string{"type": "node"}
m.meta = map[string]string{"source" : m.name, "group" : "Lustre"} m.meta = map[string]string{"source": m.name, "group": "Lustre"}
m.matches = map[string]map[string]int{"read_bytes": {"read_bytes": 6, "read_requests": 1}, m.matches = map[string]map[string]int{"read_bytes": {"read_bytes": 6, "read_requests": 1},
"write_bytes": {"write_bytes": 6, "write_requests": 1}, "write_bytes": {"write_bytes": 6, "write_requests": 1},
"open": {"open": 1}, "open": {"open": 1},
@ -90,9 +90,9 @@ func (m *LustreCollector) Read(interval time.Duration, output chan lp.CCMetric)
if err == nil { if err == nil {
y, err := lp.New(name, m.tags, m.meta, map[string]interface{}{"value": x}, time.Now()) y, err := lp.New(name, m.tags, m.meta, map[string]interface{}{"value": x}, time.Now())
if err == nil { if err == nil {
if strings.Contains(name, "byte") { if strings.Contains(name, "byte") {
y.AddMeta("unit", "Byte") y.AddMeta("unit", "Byte")
} }
output <- y output <- y
} }
} }

2
collectors/memstatMetric.go
View File

@ -35,7 +35,7 @@ func (m *MemstatCollector) Init(config json.RawMessage) error {
return err return err
} }
} }
m.meta = map[string]string{"source" : m.name, "group" : "Memory", "unit": "kByte"} m.meta = map[string]string{"source": m.name, "group": "Memory", "unit": "kByte"}
m.stats = make(map[string]int64) m.stats = make(map[string]int64)
m.matches = make(map[string]string) m.matches = make(map[string]string)
m.tags = map[string]string{"type": "node"} m.tags = map[string]string{"type": "node"}

10
collectors/metricCollector.go
View File

@ -1,6 +1,7 @@
package collectors package collectors
import ( import (
"encoding/json"
"errors" "errors"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol" influx "github.com/influxdata/line-protocol"
@ -9,7 +10,6 @@ import (
"strconv" "strconv"
"strings" "strings"
"time" "time"
"encoding/json"
) )
type MetricCollector interface { type MetricCollector interface {
@ -21,10 +21,10 @@ type MetricCollector interface {
} }
type metricCollector struct { type metricCollector struct {
output chan lp.CCMetric output chan lp.CCMetric
name string name string
init bool init bool
meta map[string]string meta map[string]string
} }
func (c *metricCollector) Name() string { func (c *metricCollector) Name() string {

14
collectors/netstatMetric.go
View File

@ -25,7 +25,7 @@ type NetstatCollector struct {
func (m *NetstatCollector) Init(config json.RawMessage) error { func (m *NetstatCollector) Init(config json.RawMessage) error {
m.name = "NetstatCollector" m.name = "NetstatCollector"
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "Memory"} m.meta = map[string]string{"source": m.name, "group": "Memory"}
m.matches = map[int]string{ m.matches = map[int]string{
1: "bytes_in", 1: "bytes_in",
9: "bytes_out", 9: "bytes_out",
@ -75,12 +75,12 @@ func (m *NetstatCollector) Read(interval time.Duration, output chan lp.CCMetric)
if err == nil { if err == nil {
y, err := lp.New(name, tags, m.meta, map[string]interface{}{"value": int(float64(v) * 1.0e-3)}, time.Now()) y, err := lp.New(name, tags, m.meta, map[string]interface{}{"value": int(float64(v) * 1.0e-3)}, time.Now())
if err == nil { if err == nil {
switch { switch {
case strings.Contains(name, "byte"): case strings.Contains(name, "byte"):
y.AddMeta("unit", "Byte") y.AddMeta("unit", "Byte")
case strings.Contains(name, "pkt"): case strings.Contains(name, "pkt"):
y.AddMeta("unit", "Packets") y.AddMeta("unit", "Packets")
} }
output <- y output <- y
} }
} }

10
collectors/nvidiaMetric.go
View File

@ -4,8 +4,8 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"github.com/NVIDIA/go-nvml/pkg/nvml"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"github.com/NVIDIA/go-nvml/pkg/nvml"
"log" "log"
"time" "time"
) )
@ -32,7 +32,7 @@ func (m *NvidiaCollector) Init(config json.RawMessage) error {
var err error var err error
m.name = "NvidiaCollector" m.name = "NvidiaCollector"
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "Nvidia"} m.meta = map[string]string{"source": m.name, "group": "Nvidia"}
if len(config) > 0 { if len(config) > 0 {
err = json.Unmarshal(config, &m.config) err = json.Unmarshal(config, &m.config)
if err != nil { if err != nil {
@ -91,14 +91,14 @@ func (m *NvidiaCollector) Read(interval time.Duration, output chan lp.CCMetric)
_, skip = stringArrayContains(m.config.ExcludeMetrics, "mem_total") _, skip = stringArrayContains(m.config.ExcludeMetrics, "mem_total")
y, err := lp.New("mem_total", tags, m.meta, map[string]interface{}{"value": t}, time.Now()) y, err := lp.New("mem_total", tags, m.meta, map[string]interface{}{"value": t}, time.Now())
if err == nil && !skip { if err == nil && !skip {
y.AddMeta("unit", "MByte") y.AddMeta("unit", "MByte")
output <- y output <- y
} }
f := float64(meminfo.Used) / (1024 * 1024) f := float64(meminfo.Used) / (1024 * 1024)
_, skip = stringArrayContains(m.config.ExcludeMetrics, "fb_memory") _, skip = stringArrayContains(m.config.ExcludeMetrics, "fb_memory")
y, err = lp.New("fb_memory", tags, m.meta, map[string]interface{}{"value": f}, time.Now()) y, err = lp.New("fb_memory", tags, m.meta, map[string]interface{}{"value": f}, time.Now())
if err == nil && !skip { if err == nil && !skip {
y.AddMeta("unit", "MByte") y.AddMeta("unit", "MByte")
output <- y output <- y
} }
} }
@ -108,7 +108,7 @@ func (m *NvidiaCollector) Read(interval time.Duration, output chan lp.CCMetric)
_, skip = stringArrayContains(m.config.ExcludeMetrics, "temp") _, skip = stringArrayContains(m.config.ExcludeMetrics, "temp")
y, err := lp.New("temp", tags, m.meta, map[string]interface{}{"value": float64(temp)}, time.Now()) y, err := lp.New("temp", tags, m.meta, map[string]interface{}{"value": float64(temp)}, time.Now())
if err == nil && !skip { if err == nil && !skip {
y.AddMeta("unit", "degC") y.AddMeta("unit", "degC")
output <- y output <- y
} }
} }

14
collectors/tempMetric.go
View File

@ -5,12 +5,12 @@ import (
"fmt" "fmt"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"io/ioutil" "io/ioutil"
"log"
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"log"
) )
const HWMON_PATH = `/sys/class/hwmon` const HWMON_PATH = `/sys/class/hwmon`
@ -28,7 +28,7 @@ type TempCollector struct {
func (m *TempCollector) Init(config json.RawMessage) error { func (m *TempCollector) Init(config json.RawMessage) error {
m.name = "TempCollector" m.name = "TempCollector"
m.setup() m.setup()
m.meta = map[string]string{"source" : m.name, "group" : "IPMI", "unit": "degC"} m.meta = map[string]string{"source": m.name, "group": "IPMI", "unit": "degC"}
if len(config) > 0 { if len(config) > 0 {
err := json.Unmarshal(config, &m.config) err := json.Unmarshal(config, &m.config)
if err != nil { if err != nil {
@ -90,10 +90,10 @@ func (m *TempCollector) Read(interval time.Duration, output chan lp.CCMetric) {
break break
} }
} }
mname := strings.Replace(name, " ", "_", -1) mname := strings.Replace(name, " ", "_", -1)
if !strings.Contains(mname, "temp") { if !strings.Contains(mname, "temp") {
mname = fmt.Sprintf("temp_%s", mname) mname = fmt.Sprintf("temp_%s", mname)
} }
buffer, err := ioutil.ReadFile(string(file)) buffer, err := ioutil.ReadFile(string(file))
if err != nil { if err != nil {
continue continue
@ -102,7 +102,7 @@ func (m *TempCollector) Read(interval time.Duration, output chan lp.CCMetric) {
if err == nil { if err == nil {
y, err := lp.New(strings.ToLower(mname), tags, m.meta, map[string]interface{}{"value": int(float64(x) / 1000)}, time.Now()) y, err := lp.New(strings.ToLower(mname), tags, m.meta, map[string]interface{}{"value": int(float64(x) / 1000)}, time.Now())
if err == nil { if err == nil {
log.Print("[", m.name, "] ",y) log.Print("[", m.name, "] ", y)
output <- y output <- y
} }
} }

2
collectors/topprocsMetric.go
View File

@ -28,7 +28,7 @@ func (m *TopProcsCollector) Init(config json.RawMessage) error {
var err error var err error
m.name = "TopProcsCollector" m.name = "TopProcsCollector"
m.tags = map[string]string{"type": "node"} m.tags = map[string]string{"type": "node"}
m.meta = map[string]string{"source" : m.name, "group" : "TopProcs"} m.meta = map[string]string{"source": m.name, "group": "TopProcs"}
if len(config) > 0 { if len(config) > 0 {
err = json.Unmarshal(config, &m.config) err = json.Unmarshal(config, &m.config)
if err != nil { if err != nil {

1
go.sum
View File

@ -1,7 +1,6 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/NVIDIA/go-nvml v0.11.1-0 h1:XHSz3zZKC4NCP2ja1rI7++DXFhA+uDhdYa3MykCTGHY= github.com/NVIDIA/go-nvml v0.11.1-0 h1:XHSz3zZKC4NCP2ja1rI7++DXFhA+uDhdYa3MykCTGHY=
github.com/NVIDIA/go-nvml v0.11.1-0/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs= github.com/NVIDIA/go-nvml v0.11.1-0/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=

25
internal/ccMetric/ccMetric.go
View File

@ -1,10 +1,10 @@
package ccmetric package ccmetric
import ( import (
lp "github.com/influxdata/line-protocol" // MIT license "fmt"
"time" lp "github.com/influxdata/line-protocol" // MIT license
"sort" "sort"
"fmt" "time"
) )
// Most functions are derived from github.com/influxdata/line-protocol/metric.go // Most functions are derived from github.com/influxdata/line-protocol/metric.go
@ -12,18 +12,18 @@ import (
// type. // type.
type ccMetric struct { type ccMetric struct {
name string name string
tags []*lp.Tag tags []*lp.Tag
fields []*lp.Field fields []*lp.Field
tm time.Time tm time.Time
meta []*lp.Tag meta []*lp.Tag
} }
type CCMetric interface { type CCMetric interface {
lp.MutableMetric lp.MutableMetric
AddMeta(key, value string) AddMeta(key, value string)
MetaList() []*lp.Tag MetaList() []*lp.Tag
RemoveTag(key string) RemoveTag(key string)
} }
func (m *ccMetric) Meta() map[string]string { func (m *ccMetric) Meta() map[string]string {
@ -187,9 +187,6 @@ func (m *ccMetric) AddField(key string, value interface{}) {
m.fields = append(m.fields, &lp.Field{Key: key, Value: convertField(value)}) m.fields = append(m.fields, &lp.Field{Key: key, Value: convertField(value)})
} }
func New( func New(
name string, name string,
tags map[string]string, tags map[string]string,
@ -202,7 +199,7 @@ func New(
tags: nil, tags: nil,
fields: nil, fields: nil,
tm: tm, tm: tm,
meta: nil, meta: nil,
} }
if len(tags) > 0 { if len(tags) > 0 {

283
internal/metricRouter/metricRouter.go
View File

@ -1,36 +1,36 @@
package metricRouter package metricRouter
import ( import (
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" "encoding/json"
"sync" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"log" mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker"
"encoding/json" "gopkg.in/Knetic/govaluate.v2"
"os" "log"
"time" "os"
"gopkg.in/Knetic/govaluate.v2" "sync"
mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker" "time"
) )
type metricRounterTagConfig struct { type metricRounterTagConfig struct {
Key string `json:"key"` Key string `json:"key"`
Value string `json:"value"` Value string `json:"value"`
Condition string `json:"if"` Condition string `json:"if"`
} }
type metricRouterConfig struct { type metricRouterConfig struct {
AddTags []metricRounterTagConfig `json:"add_tags"` AddTags []metricRounterTagConfig `json:"add_tags"`
DelTags []metricRounterTagConfig `json:"delete_tags"` DelTags []metricRounterTagConfig `json:"delete_tags"`
IntervalStamp bool `json:"interval_timestamp"` IntervalStamp bool `json:"interval_timestamp"`
} }
type metricRouter struct { type metricRouter struct {
inputs []chan lp.CCMetric inputs []chan lp.CCMetric
outputs []chan lp.CCMetric outputs []chan lp.CCMetric
done chan bool done chan bool
wg *sync.WaitGroup wg *sync.WaitGroup
timestamp time.Time timestamp time.Time
ticker mct.MultiChanTicker ticker mct.MultiChanTicker
config metricRouterConfig config metricRouterConfig
} }
type MetricRouter interface { type MetricRouter interface {
@ -41,62 +41,61 @@ type MetricRouter interface {
Close() Close()
} }
func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) error { func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) error {
r.inputs = make([]chan lp.CCMetric, 0) r.inputs = make([]chan lp.CCMetric, 0)
r.outputs = make([]chan lp.CCMetric, 0) r.outputs = make([]chan lp.CCMetric, 0)
r.done = make(chan bool) r.done = make(chan bool)
r.wg = wg r.wg = wg
r.ticker = ticker r.ticker = ticker
configFile, err := os.Open(routerConfigFile) configFile, err := os.Open(routerConfigFile)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return err return err
} }
defer configFile.Close() defer configFile.Close()
jsonParser := json.NewDecoder(configFile) jsonParser := json.NewDecoder(configFile)
err = jsonParser.Decode(&r.config) err = jsonParser.Decode(&r.config)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return err return err
} }
return nil return nil
} }
func (r *metricRouter) StartTimer() { func (r *metricRouter) StartTimer() {
m := make(chan time.Time) m := make(chan time.Time)
r.ticker.AddChannel(m) r.ticker.AddChannel(m)
go func() { go func() {
for { for {
select { select {
case t := <- m: case t := <-m:
r.timestamp = t r.timestamp = t
} }
} }
}() }()
} }
func (r *metricRouter) EvalCondition(Cond string, point lp.CCMetric) (bool, error){ func (r *metricRouter) EvalCondition(Cond string, point lp.CCMetric) (bool, error) {
expression, err := govaluate.NewEvaluableExpression(Cond) expression, err := govaluate.NewEvaluableExpression(Cond)
if err != nil { if err != nil {
log.Print(Cond, " = ", err.Error()) log.Print(Cond, " = ", err.Error())
return false, err return false, err
} }
params := make(map[string]interface{}) params := make(map[string]interface{})
params["name"] = point.Name() params["name"] = point.Name()
for _,t := range point.TagList() { for _, t := range point.TagList() {
params[t.Key] = t.Value params[t.Key] = t.Value
} }
for _,m := range point.MetaList() { for _, m := range point.MetaList() {
params[m.Key] = m.Value params[m.Key] = m.Value
} }
for _,f := range point.FieldList() { for _, f := range point.FieldList() {
params[f.Key] = f.Value params[f.Key] = f.Value
} }
params["timestamp"] = point.Time() params["timestamp"] = point.Time()
result, err := expression.Evaluate(params) result, err := expression.Evaluate(params)
if err != nil { if err != nil {
log.Print(Cond, " = ", err.Error()) log.Print(Cond, " = ", err.Error())
return false, err return false, err
} }
@ -104,106 +103,106 @@ func (r *metricRouter) EvalCondition(Cond string, point lp.CCMetric) (bool, erro
} }
func (r *metricRouter) DoAddTags(point lp.CCMetric) { func (r *metricRouter) DoAddTags(point lp.CCMetric) {
for _, m := range r.config.AddTags { for _, m := range r.config.AddTags {
var res bool var res bool
var err error var err error
if m.Condition == "*" { if m.Condition == "*" {
res = true res = true
err = nil err = nil
} else { } else {
res, err = r.EvalCondition(m.Condition, point) res, err = r.EvalCondition(m.Condition, point)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
res = false res = false
} }
} }
if res == true { if res == true {
point.AddTag(m.Key, m.Value) point.AddTag(m.Key, m.Value)
} }
} }
} }
func (r *metricRouter) DoDelTags(point lp.CCMetric) { func (r *metricRouter) DoDelTags(point lp.CCMetric) {
for _, m := range r.config.DelTags { for _, m := range r.config.DelTags {
var res bool var res bool
var err error var err error
if m.Condition == "*" { if m.Condition == "*" {
res = true res = true
err = nil err = nil
} else { } else {
res, err = r.EvalCondition(m.Condition, point) res, err = r.EvalCondition(m.Condition, point)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
res = false res = false
} }
} }
if res == true { if res == true {
point.RemoveTag(m.Key) point.RemoveTag(m.Key)
} }
} }
} }
func (r *metricRouter) Start() { func (r *metricRouter) Start() {
r.wg.Add(1) r.wg.Add(1)
r.timestamp = time.Now() r.timestamp = time.Now()
if r.config.IntervalStamp == true { if r.config.IntervalStamp == true {
r.StartTimer() r.StartTimer()
} }
go func() { go func() {
for { for {
RouterLoop: RouterLoop:
select { select {
case <- r.done: case <-r.done:
log.Print("[MetricRouter] DONE\n") log.Print("[MetricRouter] DONE\n")
r.wg.Done() r.wg.Done()
break RouterLoop break RouterLoop
default: default:
for _, c := range r.inputs { for _, c := range r.inputs {
RouterInputLoop: RouterInputLoop:
select { select {
case <- r.done: case <-r.done:
log.Print("[MetricRouter] DONE\n") log.Print("[MetricRouter] DONE\n")
r.wg.Done() r.wg.Done()
break RouterInputLoop break RouterInputLoop
case p := <- c: case p := <-c:
log.Print("[MetricRouter] FORWARD ",p) log.Print("[MetricRouter] FORWARD ", p)
r.DoAddTags(p) r.DoAddTags(p)
r.DoDelTags(p) r.DoDelTags(p)
if r.config.IntervalStamp == true { if r.config.IntervalStamp == true {
p.SetTime(r.timestamp) p.SetTime(r.timestamp)
} }
for _, o := range r.outputs { for _, o := range r.outputs {
o <- p o <- p
} }
default: default:
} }
} }
} }
} }
log.Print("[MetricRouter] EXIT\n") log.Print("[MetricRouter] EXIT\n")
}() }()
log.Print("[MetricRouter] STARTED\n") log.Print("[MetricRouter] STARTED\n")
} }
func (r *metricRouter) AddInput(input chan lp.CCMetric) { func (r *metricRouter) AddInput(input chan lp.CCMetric) {
r.inputs = append(r.inputs, input) r.inputs = append(r.inputs, input)
} }
func (r *metricRouter) AddOutput(output chan lp.CCMetric) { func (r *metricRouter) AddOutput(output chan lp.CCMetric) {
r.outputs = append(r.outputs, output) r.outputs = append(r.outputs, output)
} }
func (r *metricRouter) Close() { func (r *metricRouter) Close() {
r.done <- true r.done <- true
log.Print("[MetricRouter] CLOSE\n") log.Print("[MetricRouter] CLOSE\n")
} }
func New(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) (MetricRouter, error) { func New(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) (MetricRouter, error) {
r := &metricRouter{} r := &metricRouter{}
err := r.Init(ticker, wg, routerConfigFile) err := r.Init(ticker, wg, routerConfigFile)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return r, err return r, err
} }

40
internal/multiChanTicker/multiChanTicker.go
View File

@ -1,39 +1,39 @@
package multiChanTicker package multiChanTicker
import ( import (
"time" "time"
) )
type multiChanTicker struct { type multiChanTicker struct {
ticker *time.Ticker ticker *time.Ticker
channels []chan time.Time channels []chan time.Time
} }
type MultiChanTicker interface { type MultiChanTicker interface {
Init(duration time.Duration) Init(duration time.Duration)
AddChannel(chan time.Time) AddChannel(chan time.Time)
} }
func (t *multiChanTicker) Init(duration time.Duration) { func (t *multiChanTicker) Init(duration time.Duration) {
t.ticker = time.NewTicker(duration) t.ticker = time.NewTicker(duration)
go func() { go func() {
for { for {
select { select {
case ts := <-t.ticker.C: case ts := <-t.ticker.C:
for _, c := range t.channels { for _, c := range t.channels {
c <- ts c <- ts
} }
} }
} }
}() }()
} }
func (t *multiChanTicker) AddChannel(channel chan time.Time) { func (t *multiChanTicker) AddChannel(channel chan time.Time) {
t.channels = append(t.channels, channel) t.channels = append(t.channels, channel)
} }
func NewTicker(duration time.Duration) MultiChanTicker { func NewTicker(duration time.Duration) MultiChanTicker {
t := &multiChanTicker{} t := &multiChanTicker{}
t.Init(duration) t.Init(duration)
return t return t
} }

421
metric-collector.go
View File

@ -10,12 +10,12 @@ import (
"log" "log"
"os" "os"
"os/signal" "os/signal"
// "strings" // "strings"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
mr "github.com/ClusterCockpit/cc-metric-collector/internal/metricRouter"
mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker"
"sync" "sync"
"time" "time"
mr "github.com/ClusterCockpit/cc-metric-collector/internal/metricRouter"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker"
) )
// List of provided collectors. Which collector should be run can be // List of provided collectors. Which collector should be run can be
@ -48,17 +48,17 @@ import (
//} //}
type CentralConfigFile struct { type CentralConfigFile struct {
Interval int `json:"interval"` Interval int `json:"interval"`
Duration int `json:"duration"` Duration int `json:"duration"`
Pidfile string `json:"pidfile", omitempty` Pidfile string `json:"pidfile", omitempty`
CollectorConfigFile string `json:"collectors"` CollectorConfigFile string `json:"collectors"`
RouterConfigFile string `json:"router"` RouterConfigFile string `json:"router"`
SinkConfigFile string `json:"sinks"` SinkConfigFile string `json:"sinks"`
ReceiverConfigFile string `json:"receivers", omitempty` ReceiverConfigFile string `json:"receivers", omitempty`
} }
func LoadCentralConfiguration(file string, config *CentralConfigFile) error { func LoadCentralConfiguration(file string, config *CentralConfigFile) error {
configFile, err := os.Open(file) configFile, err := os.Open(file)
defer configFile.Close() defer configFile.Close()
if err != nil { if err != nil {
fmt.Println(err.Error()) fmt.Println(err.Error())
@ -70,29 +70,29 @@ func LoadCentralConfiguration(file string, config *CentralConfigFile) error {
} }
type RuntimeConfig struct { type RuntimeConfig struct {
Hostname string Hostname string
Interval time.Duration Interval time.Duration
Duration time.Duration Duration time.Duration
CliArgs map[string]string CliArgs map[string]string
ConfigFile CentralConfigFile ConfigFile CentralConfigFile
Router mr.MetricRouter Router mr.MetricRouter
CollectManager collectors.CollectorManager CollectManager collectors.CollectorManager
SinkManager sinks.SinkManager SinkManager sinks.SinkManager
ReceiveManager receivers.ReceiveManager ReceiveManager receivers.ReceiveManager
Ticker mct.MultiChanTicker Ticker mct.MultiChanTicker
Channels []chan lp.CCMetric Channels []chan lp.CCMetric
Sync sync.WaitGroup Sync sync.WaitGroup
} }
func prepare_runcfg() RuntimeConfig { func prepare_runcfg() RuntimeConfig {
r := RuntimeConfig{} r := RuntimeConfig{}
r.Router = nil r.Router = nil
r.CollectManager = nil r.CollectManager = nil
r.SinkManager = nil r.SinkManager = nil
r.ReceiveManager = nil r.ReceiveManager = nil
return r return r
} }
//// Structure of the configuration file //// Structure of the configuration file
@ -177,26 +177,26 @@ func ReadCli() map[string]string {
func shutdown(config *RuntimeConfig) { func shutdown(config *RuntimeConfig) {
log.Print("Shutdown...") log.Print("Shutdown...")
if config.CollectManager != nil { if config.CollectManager != nil {
log.Print("Shutdown CollectManager...") log.Print("Shutdown CollectManager...")
config.CollectManager.Close() config.CollectManager.Close()
} }
if config.ReceiveManager != nil { if config.ReceiveManager != nil {
log.Print("Shutdown ReceiveManager...") log.Print("Shutdown ReceiveManager...")
config.ReceiveManager.Close() config.ReceiveManager.Close()
} }
if config.Router != nil { if config.Router != nil {
log.Print("Shutdown Router...") log.Print("Shutdown Router...")
config.Router.Close() config.Router.Close()
} }
if config.SinkManager != nil { if config.SinkManager != nil {
log.Print("Shutdown SinkManager...") log.Print("Shutdown SinkManager...")
config.SinkManager.Close() config.SinkManager.Close()
} }
// pidfile := config.ConfigFile.Pidfile // pidfile := config.ConfigFile.Pidfile
// RemovePidfile(pidfile) // RemovePidfile(pidfile)
// pidfile = config.CliArgs["pidfile"] // pidfile = config.CliArgs["pidfile"]
// RemovePidfile(pidfile) // RemovePidfile(pidfile)
config.Sync.Done() config.Sync.Done()
} }
@ -214,7 +214,7 @@ func prepare_shutdown(config *RuntimeConfig) {
} }
func main() { func main() {
var err error var err error
use_recv := false use_recv := false
rcfg := prepare_runcfg() rcfg := prepare_runcfg()
@ -231,194 +231,193 @@ func main() {
log.Print("Configuration value 'interval' must be greater than zero") log.Print("Configuration value 'interval' must be greater than zero")
return return
} }
rcfg.Interval = time.Duration(rcfg.ConfigFile.Interval)*time.Second rcfg.Interval = time.Duration(rcfg.ConfigFile.Interval) * time.Second
if rcfg.ConfigFile.Duration <= 0 || time.Duration(rcfg.ConfigFile.Duration)*time.Second <= 0 { if rcfg.ConfigFile.Duration <= 0 || time.Duration(rcfg.ConfigFile.Duration)*time.Second <= 0 {
log.Print("Configuration value 'duration' must be greater than zero") log.Print("Configuration value 'duration' must be greater than zero")
return return
} }
rcfg.Duration = time.Duration(rcfg.ConfigFile.Duration)*time.Second rcfg.Duration = time.Duration(rcfg.ConfigFile.Duration) * time.Second
rcfg.Hostname, err = os.Hostname() rcfg.Hostname, err = os.Hostname()
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return return
} }
// err = CreatePidfile(rcfg.CliArgs["pidfile"]) // err = CreatePidfile(rcfg.CliArgs["pidfile"])
// err = SetLogging(rcfg.CliArgs["logfile"]) // err = SetLogging(rcfg.CliArgs["logfile"])
// if err != nil { // if err != nil {
// log.Print("Error setting up logging system to ", rcfg.CliArgs["logfile"], " on ", rcfg.Hostname) // log.Print("Error setting up logging system to ", rcfg.CliArgs["logfile"], " on ", rcfg.Hostname)
// return // return
// } // }
rcfg.Ticker = mct.NewTicker(rcfg.Interval) rcfg.Ticker = mct.NewTicker(rcfg.Interval)
if len(rcfg.ConfigFile.RouterConfigFile) > 0 { if len(rcfg.ConfigFile.RouterConfigFile) > 0 {
rcfg.Router, err = mr.New(rcfg.Ticker, &rcfg.Sync, rcfg.ConfigFile.RouterConfigFile) rcfg.Router, err = mr.New(rcfg.Ticker, &rcfg.Sync, rcfg.ConfigFile.RouterConfigFile)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return return
} }
} }
if len(rcfg.ConfigFile.SinkConfigFile) > 0 { if len(rcfg.ConfigFile.SinkConfigFile) > 0 {
rcfg.SinkManager, err = sinks.New(&rcfg.Sync, rcfg.ConfigFile.SinkConfigFile) rcfg.SinkManager, err = sinks.New(&rcfg.Sync, rcfg.ConfigFile.SinkConfigFile)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return return
} }
RouterToSinksChannel := make(chan lp.CCMetric) RouterToSinksChannel := make(chan lp.CCMetric)
rcfg.SinkManager.AddInput(RouterToSinksChannel) rcfg.SinkManager.AddInput(RouterToSinksChannel)
rcfg.Router.AddOutput(RouterToSinksChannel) rcfg.Router.AddOutput(RouterToSinksChannel)
} }
if len(rcfg.ConfigFile.CollectorConfigFile) > 0 { if len(rcfg.ConfigFile.CollectorConfigFile) > 0 {
rcfg.CollectManager, err = collectors.New(rcfg.Ticker, rcfg.Duration, &rcfg.Sync, rcfg.ConfigFile.CollectorConfigFile) rcfg.CollectManager, err = collectors.New(rcfg.Ticker, rcfg.Duration, &rcfg.Sync, rcfg.ConfigFile.CollectorConfigFile)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return return
} }
CollectToRouterChannel := make(chan lp.CCMetric) CollectToRouterChannel := make(chan lp.CCMetric)
rcfg.CollectManager.AddOutput(CollectToRouterChannel) rcfg.CollectManager.AddOutput(CollectToRouterChannel)
rcfg.Router.AddInput(CollectToRouterChannel) rcfg.Router.AddInput(CollectToRouterChannel)
} }
if len(rcfg.ConfigFile.ReceiverConfigFile) > 0 { if len(rcfg.ConfigFile.ReceiverConfigFile) > 0 {
rcfg.ReceiveManager, err = receivers.New(&rcfg.Sync, rcfg.ConfigFile.ReceiverConfigFile) rcfg.ReceiveManager, err = receivers.New(&rcfg.Sync, rcfg.ConfigFile.ReceiverConfigFile)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return return
} }
ReceiveToRouterChannel := make(chan lp.CCMetric) ReceiveToRouterChannel := make(chan lp.CCMetric)
rcfg.ReceiveManager.AddOutput(ReceiveToRouterChannel) rcfg.ReceiveManager.AddOutput(ReceiveToRouterChannel)
rcfg.Router.AddInput(ReceiveToRouterChannel) rcfg.Router.AddInput(ReceiveToRouterChannel)
use_recv = true use_recv = true
} }
prepare_shutdown(&rcfg) prepare_shutdown(&rcfg)
rcfg.Sync.Add(1) rcfg.Sync.Add(1)
rcfg.Router.Start() rcfg.Router.Start()
rcfg.SinkManager.Start() rcfg.SinkManager.Start()
rcfg.CollectManager.Start() rcfg.CollectManager.Start()
if use_recv { if use_recv {
rcfg.ReceiveManager.Start() rcfg.ReceiveManager.Start()
} }
// if len(config.Collectors) == 0 { // if len(config.Collectors) == 0 {
// var keys []string // var keys []string
// for k := range Collectors { // for k := range Collectors {
// keys = append(keys, k) // keys = append(keys, k)
// } // }
// log.Print("Configuration value 'collectors' does not contain any collector. Available: ", strings.Join(keys, ", ")) // log.Print("Configuration value 'collectors' does not contain any collector. Available: ", strings.Join(keys, ", "))
// return // return
// } // }
// for _, name := range config.Collectors { // for _, name := range config.Collectors {
// if _, found := Collectors[name]; !found { // if _, found := Collectors[name]; !found {
// log.Print("Invalid collector '", name, "' in configuration") // log.Print("Invalid collector '", name, "' in configuration")
// return // return
// } // }
// } // }
// if _, found := Sinks[config.Sink.Type]; !found { // if _, found := Sinks[config.Sink.Type]; !found {
// log.Print("Invalid sink type '", config.Sink.Type, "' in configuration") // log.Print("Invalid sink type '", config.Sink.Type, "' in configuration")
// return // return
// } // }
// // Setup sink // // Setup sink
// sink := Sinks[config.Sink.Type] // sink := Sinks[config.Sink.Type]
// err = sink.Init(config.Sink) // err = sink.Init(config.Sink)
// if err != nil { // if err != nil {
// log.Print(err) // log.Print(err)
// return // return
// } // }
// sinkChannel := make(chan bool) // sinkChannel := make(chan bool)
// mproxy.Init(sinkChannel, &wg) // mproxy.Init(sinkChannel, &wg)
// // Setup receiver // // Setup receiver
// if len(config.Receiver.Type) > 0 && config.Receiver.Type != "none" { // if len(config.Receiver.Type) > 0 && config.Receiver.Type != "none" {
// if _, found := Receivers[config.Receiver.Type]; !found { // if _, found := Receivers[config.Receiver.Type]; !found {
// log.Print("Invalid receiver type '", config.Receiver.Type, "' in configuration") // log.Print("Invalid receiver type '", config.Receiver.Type, "' in configuration")
// return // return
// } else { // } else {
// recv = Receivers[config.Receiver.Type] // recv = Receivers[config.Receiver.Type]
// err = recv.Init(config.Receiver, sink) // err = recv.Init(config.Receiver, sink)
// if err == nil { // if err == nil {
// use_recv = true // use_recv = true
// } else { // } else {
// log.Print(err) // log.Print(err)
// } // }
// } // }
// } // }
// // Register interrupt handler // // Register interrupt handler
// prepare_shutdown(&wg, &config, sink, recv, clicfg["pidfile"]) // prepare_shutdown(&wg, &config, sink, recv, clicfg["pidfile"])
// // Initialize all collectors // // Initialize all collectors
// tmp := make([]string, 0) // tmp := make([]string, 0)
// for _, c := range config.Collectors { // for _, c := range config.Collectors {
// col := Collectors[c] // col := Collectors[c]
// conf, found := config.CollectConfigs[c] // conf, found := config.CollectConfigs[c]
// if !found { // if !found {
// conf = json.RawMessage("") // conf = json.RawMessage("")
// } // }
// err = col.Init([]byte(conf)) // err = col.Init([]byte(conf))
// if err != nil { // if err != nil {
// log.Print("SKIP ", col.Name(), " (", err.Error(), ")") // log.Print("SKIP ", col.Name(), " (", err.Error(), ")")
// } else if !col.Initialized() { // } else if !col.Initialized() {
// log.Print("SKIP ", col.Name(), " (Not initialized)") // log.Print("SKIP ", col.Name(), " (Not initialized)")
// } else { // } else {
// log.Print("Start ", col.Name()) // log.Print("Start ", col.Name())
// tmp = append(tmp, c) // tmp = append(tmp, c)
// } // }
// } // }
// config.Collectors = tmp // config.Collectors = tmp
// config.DefTags["hostname"] = host // config.DefTags["hostname"] = host
// // Setup up ticker loop // // Setup up ticker loop
// if clicfg["once"] != "true" { // if clicfg["once"] != "true" {
// log.Print("Running loop every ", time.Duration(config.Interval)*time.Second) // log.Print("Running loop every ", time.Duration(config.Interval)*time.Second)
// } else { // } else {
// log.Print("Running loop only once") // log.Print("Running loop only once")
// } // }
// ticker := time.NewTicker(time.Duration(config.Interval) * time.Second) // ticker := time.NewTicker(time.Duration(config.Interval) * time.Second)
// done := make(chan bool) // done := make(chan bool)
// // Storage for all node metrics // // Storage for all node metrics
// tmpPoints := make([]lp.MutableMetric, 0) // tmpPoints := make([]lp.MutableMetric, 0)
// // Start receiver // // Start receiver
// if use_recv { // if use_recv {
// recv.Start() // recv.Start()
// } // }
// go func() { // go func() {
// for { // for {
// select { // select {
// case <-done: // case <-done:
// return // return
// case t := <-ticker.C: // case t := <-ticker.C:
// // Read all collectors are sort the results in the right // // Read all collectors are sort the results in the right
// // storage locations // // storage locations
// for _, c := range config.Collectors { // for _, c := range config.Collectors {
// col := Collectors[c] // col := Collectors[c]
// col.Read(time.Duration(config.Duration), &tmpPoints) // col.Read(time.Duration(config.Duration), &tmpPoints)
// for { // for {
// if len(tmpPoints) == 0 { // if len(tmpPoints) == 0 {
// break // break
// } // }
// p := tmpPoints[0] // p := tmpPoints[0]
// for k, v := range config.DefTags { // for k, v := range config.DefTags {
// p.AddTag(k, v) // p.AddTag(k, v)
// p.SetTime(t) // p.SetTime(t)
// } // }
// sink.Write(p) // sink.Write(p)
// tmpPoints = tmpPoints[1:] // tmpPoints = tmpPoints[1:]
// } // }
// } // }
// if err := sink.Flush(); err != nil { // if err := sink.Flush(); err != nil {
// log.Printf("sink error: %s\n", err) // log.Printf("sink error: %s\n", err)
// } // }
// if clicfg["once"] == "true" { // if clicfg["once"] == "true" {
// shutdown(&wg, config.Collectors, sink, recv, clicfg["pidfile"]) // shutdown(&wg, config.Collectors, sink, recv, clicfg["pidfile"])
// return // return
// } // }
// } // }
// } // }
// }() // }()
// Wait until receiving an interrupt // Wait until receiving an interrupt
rcfg.Sync.Wait() rcfg.Sync.Wait()

13
receivers/metricReceiver.go
View File

@ -2,17 +2,16 @@ package receivers
import ( import (
// "time" // "time"
influx "github.com/influxdata/line-protocol"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol"
) )
type ReceiverConfig struct { type ReceiverConfig struct {
Addr string `json:"address"` Addr string `json:"address"`
Port string `json:"port"` Port string `json:"port"`
Database string `json:"database"` Database string `json:"database"`
Organization string `json:"organization", omitempty` Organization string `json:"organization", omitempty`
Type string `json:"type"` Type string `json:"type"`
} }
type receiver struct { type receiver struct {
@ -37,7 +36,7 @@ func (r *receiver) Name() string {
} }
func (r *receiver) SetSink(sink chan lp.CCMetric) { func (r *receiver) SetSink(sink chan lp.CCMetric) {
r.sink = sink r.sink = sink
} }
func Tags2Map(metric influx.Metric) map[string]string { func Tags2Map(metric influx.Metric) map[string]string {

17
receivers/natsReceiver.go
View File

@ -2,16 +2,16 @@ package receivers
import ( import (
"errors" "errors"
influx "github.com/influxdata/line-protocol" "fmt"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol"
nats "github.com/nats-io/nats.go" nats "github.com/nats-io/nats.go"
"log" "log"
"time" "time"
"fmt"
) )
type NatsReceiverConfig struct { type NatsReceiverConfig struct {
Addr string `json:"address"` Addr string `json:"address"`
Port string `json:"port"` Port string `json:"port"`
Database string `json:"database"` Database string `json:"database"`
} }
@ -37,7 +37,7 @@ func (r *NatsReceiver) Init(config ReceiverConfig) error {
len(r.config.Database) == 0 { len(r.config.Database) == 0 {
return errors.New("Not all configuration variables set required by NatsReceiver") return errors.New("Not all configuration variables set required by NatsReceiver")
} }
r.meta = map[string]string{"source" : r.name} r.meta = map[string]string{"source": r.name}
r.addr = r.config.Addr r.addr = r.config.Addr
if len(r.addr) == 0 { if len(r.addr) == 0 {
r.addr = nats.DefaultURL r.addr = nats.DefaultURL
@ -71,10 +71,10 @@ func (r *NatsReceiver) _NatsReceive(m *nats.Msg) {
metrics, err := r.parser.Parse(m.Data) metrics, err := r.parser.Parse(m.Data)
if err == nil { if err == nil {
for _, m := range metrics { for _, m := range metrics {
y := lp.FromInfluxMetric(m) y := lp.FromInfluxMetric(m)
for k, v := range r.meta { for k, v := range r.meta {
y.AddMeta(k, v) y.AddMeta(k, v)
} }
//y, err := lp.New(m.Name(), Tags2Map(m), r.meta, Fields2Map(m), m.Time()) //y, err := lp.New(m.Name(), Tags2Map(m), r.meta, Fields2Map(m), m.Time())
if r.sink != nil { if r.sink != nil {
r.sink <- y r.sink <- y
@ -89,4 +89,3 @@ func (r *NatsReceiver) Close() {
r.nc.Close() r.nc.Close()
} }
} }

223
receivers/receiveManager.go
View File

@ -1,24 +1,23 @@
package receivers package receivers
import ( import (
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" "encoding/json"
"sync" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"log" "log"
"os" "os"
"encoding/json" "sync"
) )
var AvailableReceivers = map[string]Receiver{ var AvailableReceivers = map[string]Receiver{
"nats": &NatsReceiver{}, "nats": &NatsReceiver{},
} }
type receiveManager struct { type receiveManager struct {
inputs []Receiver inputs []Receiver
output chan lp.CCMetric output chan lp.CCMetric
done chan bool done chan bool
wg *sync.WaitGroup wg *sync.WaitGroup
config []ReceiverConfig config []ReceiverConfig
} }
type ReceiveManager interface { type ReceiveManager interface {
@ -29,128 +28,126 @@ type ReceiveManager interface {
Close() Close()
} }
func (rm *receiveManager) Init(wg *sync.WaitGroup, receiverConfigFile string) error { func (rm *receiveManager) Init(wg *sync.WaitGroup, receiverConfigFile string) error {
rm.inputs = make([]Receiver, 0) rm.inputs = make([]Receiver, 0)
rm.output = nil rm.output = nil
rm.done = make(chan bool) rm.done = make(chan bool)
rm.wg = wg rm.wg = wg
rm.config = make([]ReceiverConfig, 0) rm.config = make([]ReceiverConfig, 0)
configFile, err := os.Open(receiverConfigFile) configFile, err := os.Open(receiverConfigFile)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return err return err
} }
defer configFile.Close() defer configFile.Close()
jsonParser := json.NewDecoder(configFile) jsonParser := json.NewDecoder(configFile)
var rawConfigs []json.RawMessage var rawConfigs []json.RawMessage
err = jsonParser.Decode(&rawConfigs) err = jsonParser.Decode(&rawConfigs)
if err != nil { if err != nil {
log.Print(err.Error()) log.Print(err.Error())
return err return err
} }
for _, raw := range rawConfigs { for _, raw := range rawConfigs {
log.Print("[ReceiveManager] ", string(raw)) log.Print("[ReceiveManager] ", string(raw))
rm.AddInput(raw) rm.AddInput(raw)
// if _, found := AvailableReceivers[k.Type]; !found { // if _, found := AvailableReceivers[k.Type]; !found {
// log.Print("[ReceiveManager] SKIP Config specifies unknown receiver 'type': ", k.Type) // log.Print("[ReceiveManager] SKIP Config specifies unknown receiver 'type': ", k.Type)
// continue // continue
// } // }
// r := AvailableReceivers[k.Type] // r := AvailableReceivers[k.Type]
// err = r.Init(k) // err = r.Init(k)
// if err != nil { // if err != nil {
// log.Print("[ReceiveManager] SKIP Receiver ", k.Type, " cannot be initialized: ", err.Error()) // log.Print("[ReceiveManager] SKIP Receiver ", k.Type, " cannot be initialized: ", err.Error())
// continue // continue
// } // }
// rm.inputs = append(rm.inputs, r) // rm.inputs = append(rm.inputs, r)
} }
return nil return nil
} }
func (rm *receiveManager) Start() { func (rm *receiveManager) Start() {
rm.wg.Add(1) rm.wg.Add(1)
for _, r := range rm.inputs { for _, r := range rm.inputs {
log.Print("[ReceiveManager] START ", r.Name()) log.Print("[ReceiveManager] START ", r.Name())
r.Start() r.Start()
} }
log.Print("[ReceiveManager] STARTED\n") log.Print("[ReceiveManager] STARTED\n")
// go func() { // go func() {
// for { // for {
//ReceiveManagerLoop: //ReceiveManagerLoop:
// select { // select {
// case <- rm.done: // case <- rm.done:
// log.Print("ReceiveManager done\n") // log.Print("ReceiveManager done\n")
// rm.wg.Done() // rm.wg.Done()
// break ReceiveManagerLoop // break ReceiveManagerLoop
// default: // default:
// for _, c := range rm.inputs { // for _, c := range rm.inputs {
//ReceiveManagerInputLoop: //ReceiveManagerInputLoop:
// select { // select {
// case <- rm.done: // case <- rm.done:
// log.Print("ReceiveManager done\n") // log.Print("ReceiveManager done\n")
// rm.wg.Done() // rm.wg.Done()
// break ReceiveManagerInputLoop // break ReceiveManagerInputLoop
// case p := <- c: // case p := <- c:
// log.Print("ReceiveManager: ", p) // log.Print("ReceiveManager: ", p)
// rm.output <- p // rm.output <- p
// default: // default:
// } // }
// } // }
// } // }
// } // }
// }() // }()
// for _, r := range rm.inputs { // for _, r := range rm.inputs {
// r.Close() // r.Close()
// } // }
} }
func (rm *receiveManager) AddInput(rawConfig json.RawMessage) error { func (rm *receiveManager) AddInput(rawConfig json.RawMessage) error {
var config ReceiverConfig var config ReceiverConfig
err := json.Unmarshal(rawConfig, &config) err := json.Unmarshal(rawConfig, &config)
if err != nil { if err != nil {
log.Print("[ReceiveManager] SKIP ", config.Type, " JSON config error: ", err.Error()) log.Print("[ReceiveManager] SKIP ", config.Type, " JSON config error: ", err.Error())
log.Print(err.Error()) log.Print(err.Error())
return err return err
} }
if _, found := AvailableReceivers[config.Type]; !found { if _, found := AvailableReceivers[config.Type]; !found {
log.Print("[ReceiveManager] SKIP ", config.Type, " unknown receiver: ", err.Error()) log.Print("[ReceiveManager] SKIP ", config.Type, " unknown receiver: ", err.Error())
return err return err
}
r := AvailableReceivers[config.Type]
err = r.Init(config)
if err != nil {
log.Print("[ReceiveManager] SKIP ", r.Name(), " initialization failed: ", err.Error())
return err
} }
rm.inputs = append(rm.inputs, r) r := AvailableReceivers[config.Type]
rm.config = append(rm.config, config) err = r.Init(config)
return nil if err != nil {
log.Print("[ReceiveManager] SKIP ", r.Name(), " initialization failed: ", err.Error())
return err
}
rm.inputs = append(rm.inputs, r)
rm.config = append(rm.config, config)
return nil
} }
func (rm *receiveManager) AddOutput(output chan lp.CCMetric) { func (rm *receiveManager) AddOutput(output chan lp.CCMetric) {
rm.output = output rm.output = output
for _, r := range rm.inputs { for _, r := range rm.inputs {
r.SetSink(rm.output) r.SetSink(rm.output)
} }
} }
func (rm *receiveManager) Close() { func (rm *receiveManager) Close() {
for _, r := range rm.inputs { for _, r := range rm.inputs {
log.Print("[ReceiveManager] CLOSE ", r.Name()) log.Print("[ReceiveManager] CLOSE ", r.Name())
r.Close() r.Close()
} }
rm.wg.Done() rm.wg.Done()
log.Print("[ReceiveManager] CLOSE\n") log.Print("[ReceiveManager] CLOSE\n")
log.Print("[ReceiveManager] EXIT\n") log.Print("[ReceiveManager] EXIT\n")
} }
func New(wg *sync.WaitGroup, receiverConfigFile string) (ReceiveManager, error) { func New(wg *sync.WaitGroup, receiverConfigFile string) (ReceiveManager, error) {
r := &receiveManager{} r := &receiveManager{}
err := r.Init(wg, receiverConfigFile) err := r.Init(wg, receiverConfigFile)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return r, err return r, err
} }

4
sinks/httpSink.go
View File

@ -7,8 +7,8 @@ import (
"net/http" "net/http"
"time" "time"
influx "github.com/influxdata/line-protocol"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol"
) )
type HttpSink struct { type HttpSink struct {
@ -20,7 +20,7 @@ type HttpSink struct {
} }
func (s *HttpSink) Init(config sinkConfig) error { func (s *HttpSink) Init(config sinkConfig) error {
s.name = "HttpSink" s.name = "HttpSink"
if len(config.Host) == 0 || len(config.Port) == 0 || len(config.Database) == 0 { if len(config.Host) == 0 || len(config.Port) == 0 || len(config.Database) == 0 {
return errors.New("`host`, `port` and `database` config options required for TCP sink") return errors.New("`host`, `port` and `database` config options required for TCP sink")
} }

8
sinks/influxSink.go
View File

@ -39,7 +39,7 @@ func (s *InfluxSink) connect() error {
} }
func (s *InfluxSink) Init(config sinkConfig) error { func (s *InfluxSink) Init(config sinkConfig) error {
s.name = "InfluxSink" s.name = "InfluxSink"
if len(config.Host) == 0 || if len(config.Host) == 0 ||
len(config.Port) == 0 || len(config.Port) == 0 ||
len(config.Database) == 0 || len(config.Database) == 0 ||
@ -65,9 +65,9 @@ func (s *InfluxSink) Write(point lp.CCMetric) error {
tags[t.Key] = t.Value tags[t.Key] = t.Value
} }
if s.meta_as_tags { if s.meta_as_tags {
for _, m := range point.MetaList() { for _, m := range point.MetaList() {
tags[m.Key] = m.Value tags[m.Key] = m.Value
} }
} }
for _, f := range point.FieldList() { for _, f := range point.FieldList() {
fields[f.Key] = f.Value fields[f.Key] = f.Value

2
sinks/metricSink.go
View File

@ -6,7 +6,7 @@ import (
) )
type sinkConfig struct { type sinkConfig struct {
Type string `json:"type"` Type string `json:"type"`
Host string `json:"host", omitempty` Host string `json:"host", omitempty`
Port string `json:"port", omitempty` Port string `json:"port", omitempty`
Database string `json:"database, omitempty"` Database string `json:"database, omitempty"`

4
sinks/natsSink.go
View File

@ -4,8 +4,8 @@ import (
"bytes" "bytes"
"errors" "errors"
"fmt" "fmt"
influx "github.com/influxdata/line-protocol"
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
influx "github.com/influxdata/line-protocol"
nats "github.com/nats-io/nats.go" nats "github.com/nats-io/nats.go"
"log" "log"
"time" "time"
@ -33,7 +33,7 @@ func (s *NatsSink) connect() error {
} }
func (s *NatsSink) Init(config sinkConfig) error { func (s *NatsSink) Init(config sinkConfig) error {
s.name = "NatsSink" s.name = "NatsSink"
if len(config.Host) == 0 || if len(config.Host) == 0 ||
len(config.Port) == 0 || len(config.Port) == 0 ||
len(config.Database) == 0 { len(config.Database) == 0 {

211
sinks/sinkManager.go
View File

@ -1,17 +1,16 @@
package sinks package sinks
import ( import (
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" "encoding/json"
"sync" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
"log" "log"
"os" "os"
"encoding/json" "sync"
) )
type SinkEntity struct { type SinkEntity struct {
config json.RawMessage config json.RawMessage
output Sink output Sink
} }
var AvailableSinks = map[string]Sink{ var AvailableSinks = map[string]Sink{
@ -21,13 +20,12 @@ var AvailableSinks = map[string]Sink{
"http": &HttpSink{}, "http": &HttpSink{},
} }
type sinkManager struct { type sinkManager struct {
input chan lp.CCMetric input chan lp.CCMetric
outputs []Sink outputs []Sink
done chan bool done chan bool
wg *sync.WaitGroup wg *sync.WaitGroup
config []sinkConfig config []sinkConfig
} }
type SinkManager interface { type SinkManager interface {
@ -38,114 +36,111 @@ type SinkManager interface {
Close() Close()
} }
func (sm *sinkManager) Init(wg *sync.WaitGroup, sinkConfigFile string) error { func (sm *sinkManager) Init(wg *sync.WaitGroup, sinkConfigFile string) error {
sm.input = nil sm.input = nil
sm.outputs = make([]Sink, 0) sm.outputs = make([]Sink, 0)
sm.done = make(chan bool) sm.done = make(chan bool)
sm.wg = wg sm.wg = wg
sm.config = make([]sinkConfig, 0) sm.config = make([]sinkConfig, 0)
if len(sinkConfigFile) > 0 { if len(sinkConfigFile) > 0 {
configFile, err := os.Open(sinkConfigFile) configFile, err := os.Open(sinkConfigFile)
if err != nil { if err != nil {
log.Print("[SinkManager] ", err.Error()) log.Print("[SinkManager] ", err.Error())
return err return err
} }
defer configFile.Close() defer configFile.Close()
jsonParser := json.NewDecoder(configFile) jsonParser := json.NewDecoder(configFile)
var rawConfigs []json.RawMessage var rawConfigs []json.RawMessage
err = jsonParser.Decode(&rawConfigs) err = jsonParser.Decode(&rawConfigs)
if err != nil { if err != nil {
log.Print("[SinkManager] ", err.Error()) log.Print("[SinkManager] ", err.Error())
return err return err
} }
for _, raw := range rawConfigs { for _, raw := range rawConfigs {
err = sm.AddOutput(raw) err = sm.AddOutput(raw)
if err != nil { if err != nil {
continue continue
} }
} }
} }
return nil return nil
} }
func (sm *sinkManager) Start() { func (sm *sinkManager) Start() {
sm.wg.Add(1) sm.wg.Add(1)
batchcount := 20 batchcount := 20
go func() { go func() {
for { for {
SinkManagerLoop: SinkManagerLoop:
select { select {
case <- sm.done: case <-sm.done:
for _, s := range sm.outputs { for _, s := range sm.outputs {
s.Close() s.Close()
} }
log.Print("[SinkManager] DONE\n") log.Print("[SinkManager] DONE\n")
sm.wg.Done() sm.wg.Done()
break SinkManagerLoop break SinkManagerLoop
case p := <- sm.input: case p := <-sm.input:
log.Print("[SinkManager] WRITE ", p) log.Print("[SinkManager] WRITE ", p)
for _, s := range sm.outputs { for _, s := range sm.outputs {
s.Write(p) s.Write(p)
} }
if (batchcount == 0) { if batchcount == 0 {
log.Print("[SinkManager] FLUSH") log.Print("[SinkManager] FLUSH")
for _, s := range sm.outputs { for _, s := range sm.outputs {
s.Flush() s.Flush()
} }
batchcount = 20 batchcount = 20
} }
batchcount-- batchcount--
default: default:
} }
} }
log.Print("[SinkManager] EXIT\n") log.Print("[SinkManager] EXIT\n")
}() }()
log.Print("[SinkManager] STARTED\n") log.Print("[SinkManager] STARTED\n")
} }
func (sm *sinkManager) AddInput(input chan lp.CCMetric) { func (sm *sinkManager) AddInput(input chan lp.CCMetric) {
sm.input = input sm.input = input
} }
func (sm *sinkManager) AddOutput(rawConfig json.RawMessage) error { func (sm *sinkManager) AddOutput(rawConfig json.RawMessage) error {
var err error var err error
var config sinkConfig var config sinkConfig
if len(rawConfig) > 3 { if len(rawConfig) > 3 {
err = json.Unmarshal(rawConfig, &config) err = json.Unmarshal(rawConfig, &config)
if err != nil { if err != nil {
log.Print("[SinkManager] SKIP ", config.Type, " JSON config error: ", err.Error()) log.Print("[SinkManager] SKIP ", config.Type, " JSON config error: ", err.Error())
return err return err
} }
} }
if _, found := AvailableSinks[config.Type]; !found { if _, found := AvailableSinks[config.Type]; !found {
log.Print("[SinkManager] SKIP ", config.Type, " unknown sink: ", err.Error()) log.Print("[SinkManager] SKIP ", config.Type, " unknown sink: ", err.Error())
return err return err
} }
s := AvailableSinks[config.Type] s := AvailableSinks[config.Type]
err = s.Init(config) err = s.Init(config)
if err != nil { if err != nil {
log.Print("[SinkManager] SKIP ", s.Name(), " initialization failed: ", err.Error()) log.Print("[SinkManager] SKIP ", s.Name(), " initialization failed: ", err.Error())
return err return err
} }
sm.outputs = append(sm.outputs, s) sm.outputs = append(sm.outputs, s)
sm.config = append(sm.config, config) sm.config = append(sm.config, config)
return nil return nil
} }
func (sm *sinkManager) Close() { func (sm *sinkManager) Close() {
sm.done <- true sm.done <- true
log.Print("[SinkManager] CLOSE") log.Print("[SinkManager] CLOSE")
return return
} }
func New(wg *sync.WaitGroup, sinkConfigFile string) (SinkManager, error) { func New(wg *sync.WaitGroup, sinkConfigFile string) (SinkManager, error) {
sm := &sinkManager{} sm := &sinkManager{}
err := sm.Init(wg, sinkConfigFile) err := sm.Init(wg, sinkConfigFile)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return sm, err return sm, err
} }

10
sinks/stdoutSink.go
View File

@ -14,8 +14,8 @@ type StdoutSink struct {
} }
func (s *StdoutSink) Init(config sinkConfig) error { func (s *StdoutSink) Init(config sinkConfig) error {
s.name = "StdoutSink" s.name = "StdoutSink"
s.meta_as_tags = config.MetaAsTags s.meta_as_tags = config.MetaAsTags
return nil return nil
} }
@ -26,9 +26,9 @@ func (s *StdoutSink) Write(point lp.CCMetric) error {
tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", t.Key, t.Value)) tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", t.Key, t.Value))
} }
if s.meta_as_tags { if s.meta_as_tags {
for _, m := range point.MetaList() { for _, m := range point.MetaList() {
tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", m.Key, m.Value)) tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", m.Key, m.Value))
} }
} }
for _, f := range point.FieldList() { for _, f := range point.FieldList() {
switch f.Value.(type) { switch f.Value.(type) {