cc-metric-collector/metric-collector.go

317 lines
8.2 KiB
Go
Raw Normal View History

2021-03-25 14:46:25 +01:00
package main
import (
"encoding/json"
"flag"
2021-03-25 14:46:25 +01:00
"fmt"
"github.com/ClusterCockpit/cc-metric-collector/collectors"
"github.com/ClusterCockpit/cc-metric-collector/receivers"
"github.com/ClusterCockpit/cc-metric-collector/sinks"
2021-10-04 15:23:43 +02:00
lp "github.com/influxdata/line-protocol"
2021-03-25 14:46:25 +01:00
"log"
"os"
"os/signal"
"strings"
2021-03-25 14:46:25 +01:00
"sync"
"time"
)
// List of provided collectors. Which collector should be run can be
// configured at 'collectors' list in 'config.json'.
2021-03-25 14:46:25 +01:00
var Collectors = map[string]collectors.MetricGetter{
2021-03-25 17:47:08 +01:00
"likwid": &collectors.LikwidCollector{},
"loadavg": &collectors.LoadavgCollector{},
"memstat": &collectors.MemstatCollector{},
"netstat": &collectors.NetstatCollector{},
"ibstat": &collectors.InfinibandCollector{},
"lustrestat": &collectors.LustreCollector{},
"cpustat": &collectors.CpustatCollector{},
"topprocs": &collectors.TopProcsCollector{},
"nvidia": &collectors.NvidiaCollector{},
"customcmd": &collectors.CustomCmdCollector{},
"diskstat": &collectors.DiskstatCollector{},
"tempstat": &collectors.TempCollector{},
2021-11-29 15:32:58 +01:00
"ipmistat": &collectors.IpmiCollector{},
"gpfs": &collectors.GpfsCollector{},
2021-03-25 14:46:25 +01:00
}
2021-03-25 17:47:08 +01:00
var Sinks = map[string]sinks.SinkFuncs{
"influxdb": &sinks.InfluxSink{},
"stdout": &sinks.StdoutSink{},
"nats": &sinks.NatsSink{},
"http": &sinks.HttpSink{},
}
var Receivers = map[string]receivers.ReceiverFuncs{
"nats": &receivers.NatsReceiver{},
}
// Structure of the configuration file
2021-03-25 14:46:25 +01:00
type GlobalConfig struct {
2021-11-25 15:11:39 +01:00
Sink sinks.SinkConfig `json:"sink"`
Interval int `json:"interval"`
Duration int `json:"duration"`
Collectors []string `json:"collectors"`
Receiver receivers.ReceiverConfig `json:"receiver"`
DefTags map[string]string `json:"default_tags"`
CollectConfigs map[string]json.RawMessage `json:"collect_config"`
2021-03-25 14:46:25 +01:00
}
// Load JSON configuration file
2021-03-25 14:46:25 +01:00
func LoadConfiguration(file string, config *GlobalConfig) error {
configFile, err := os.Open(file)
defer configFile.Close()
if err != nil {
fmt.Println(err.Error())
2021-05-11 12:41:29 +02:00
return err
2021-03-25 14:46:25 +01:00
}
jsonParser := json.NewDecoder(configFile)
err = jsonParser.Decode(config)
2021-03-25 14:46:25 +01:00
return err
}
2021-05-11 13:16:35 +02:00
func ReadCli() map[string]string {
var m map[string]string
cfg := flag.String("config", "./config.json", "Path to configuration file")
logfile := flag.String("log", "stderr", "Path for logfile")
pidfile := flag.String("pidfile", "/var/run/cc-metric-collector.pid", "Path for PID file")
once := flag.Bool("once", false, "Run all collectors only once")
flag.Parse()
m = make(map[string]string)
m["configfile"] = *cfg
m["logfile"] = *logfile
m["pidfile"] = *pidfile
if *once {
2021-11-25 15:11:39 +01:00
m["once"] = "true"
} else {
2021-11-25 15:11:39 +01:00
m["once"] = "false"
}
return m
2021-05-11 13:16:35 +02:00
}
func SetLogging(logfile string) error {
var file *os.File
var err error
if logfile != "stderr" {
file, err = os.OpenFile(logfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
log.Fatal(err)
return err
}
} else {
file = os.Stderr
}
log.SetOutput(file)
return nil
2021-05-11 12:41:29 +02:00
}
func CreatePidfile(pidfile string) error {
2021-10-04 15:23:43 +02:00
file, err := os.OpenFile(pidfile, os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
log.Print(err)
return err
}
file.Write([]byte(fmt.Sprintf("%d", os.Getpid())))
file.Close()
return nil
}
func RemovePidfile(pidfile string) error {
2021-10-04 15:23:43 +02:00
info, err := os.Stat(pidfile)
if !os.IsNotExist(err) && !info.IsDir() {
os.Remove(pidfile)
}
return nil
}
// General shutdown function that gets executed in case of interrupt or graceful shutdown
func shutdown(wg *sync.WaitGroup, collectors []string, sink sinks.SinkFuncs, recv receivers.ReceiverFuncs, pidfile string) {
2021-11-25 15:11:39 +01:00
log.Print("Shutdown...")
for _, c := range collectors {
col := Collectors[c]
log.Print("Stop ", col.Name())
col.Close()
}
time.Sleep(1 * time.Second)
if recv != nil {
recv.Close()
}
sink.Close()
RemovePidfile(pidfile)
wg.Done()
}
// Register an interrupt handler for Ctrl+C and similar. At signal,
// all collectors are closed
func prepare_shutdown(wg *sync.WaitGroup, config *GlobalConfig, sink sinks.SinkFuncs, recv receivers.ReceiverFuncs, pidfile string) {
2021-03-25 14:46:25 +01:00
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, os.Interrupt)
go func(wg *sync.WaitGroup) {
<-sigs
log.Print("Shutdown...")
shutdown(wg, config.Collectors, sink, recv, pidfile)
2021-03-25 14:46:25 +01:00
}(wg)
}
func main() {
var config GlobalConfig
var wg sync.WaitGroup
var recv receivers.ReceiverFuncs = nil
var use_recv bool
use_recv = false
2021-03-25 14:46:25 +01:00
wg.Add(1)
host, err := os.Hostname()
2021-03-25 17:47:08 +01:00
if err != nil {
log.Print(err)
return
}
2022-01-19 15:35:12 +01:00
// Drop domain part of host name
host = strings.SplitN(host, `.`, 2)[0]
2021-05-11 13:16:35 +02:00
clicfg := ReadCli()
err = CreatePidfile(clicfg["pidfile"])
2021-05-11 13:16:35 +02:00
err = SetLogging(clicfg["logfile"])
if err != nil {
2021-10-04 15:23:43 +02:00
log.Print("Error setting up logging system to ", clicfg["logfile"], " on ", host)
return
2021-05-11 13:16:35 +02:00
}
2021-03-25 14:46:25 +01:00
// Load and check configuration
2021-05-11 13:16:35 +02:00
err = LoadConfiguration(clicfg["configfile"], &config)
if err != nil {
log.Print("Error reading configuration file ", clicfg["configfile"])
log.Print(err.Error())
return
2021-05-11 12:41:29 +02:00
}
2021-03-25 17:47:08 +01:00
if config.Interval <= 0 || time.Duration(config.Interval)*time.Second <= 0 {
log.Print("Configuration value 'interval' must be greater than zero")
return
2021-03-25 14:46:25 +01:00
}
if config.Duration <= 0 {
2021-03-25 17:47:08 +01:00
log.Print("Configuration value 'duration' must be greater than zero")
return
2021-03-25 14:46:25 +01:00
}
if len(config.Collectors) == 0 {
var keys []string
for k := range Collectors {
keys = append(keys, k)
}
log.Print("Configuration value 'collectors' does not contain any collector. Available: ", strings.Join(keys, ", "))
return
}
for _, name := range config.Collectors {
if _, found := Collectors[name]; !found {
log.Print("Invalid collector '", name, "' in configuration")
return
}
}
if _, found := Sinks[config.Sink.Type]; !found {
log.Print("Invalid sink type '", config.Sink.Type, "' in configuration")
return
}
// Setup sink
sink := Sinks[config.Sink.Type]
err = sink.Init(config.Sink)
if err != nil {
log.Print(err)
return
}
// Setup receiver
if len(config.Receiver.Type) > 0 && config.Receiver.Type != "none" {
if _, found := Receivers[config.Receiver.Type]; !found {
log.Print("Invalid receiver type '", config.Receiver.Type, "' in configuration")
return
} else {
recv = Receivers[config.Receiver.Type]
err = recv.Init(config.Receiver, sink)
if err == nil {
use_recv = true
} else {
log.Print(err)
}
}
}
// Register interrupt handler
prepare_shutdown(&wg, &config, sink, recv, clicfg["pidfile"])
// Initialize all collectors
tmp := make([]string, 0)
2021-03-25 14:46:25 +01:00
for _, c := range config.Collectors {
col := Collectors[c]
conf, found := config.CollectConfigs[c]
if !found {
2021-11-25 15:11:39 +01:00
conf = json.RawMessage("")
}
err = col.Init([]byte(conf))
if err != nil {
2021-11-25 15:11:39 +01:00
log.Print("SKIP ", col.Name(), " (", err.Error(), ")")
} else if !col.Initialized() {
2021-11-29 15:32:58 +01:00
log.Print("SKIP ", col.Name(), " (Not initialized)")
} else {
log.Print("Start ", col.Name())
tmp = append(tmp, c)
}
2021-03-25 14:46:25 +01:00
}
config.Collectors = tmp
config.DefTags["hostname"] = host
// Setup up ticker loop
if clicfg["once"] != "true" {
2021-11-25 15:11:39 +01:00
log.Print("Running loop every ", time.Duration(config.Interval)*time.Second)
} else {
log.Print("Running loop only once")
}
2021-03-25 14:46:25 +01:00
ticker := time.NewTicker(time.Duration(config.Interval) * time.Second)
done := make(chan bool)
// Storage for all node metrics
2021-10-04 15:23:43 +02:00
tmpPoints := make([]lp.MutableMetric, 0)
2021-03-25 14:46:25 +01:00
// Start receiver
if use_recv {
recv.Start()
}
2021-03-25 14:46:25 +01:00
go func() {
for {
select {
case <-done:
return
2021-03-25 17:47:08 +01:00
case t := <-ticker.C:
// Read all collectors are sort the results in the right
// storage locations
2021-03-25 17:47:08 +01:00
for _, c := range config.Collectors {
col := Collectors[c]
col.Read(time.Duration(config.Duration)*time.Second, &tmpPoints)
2021-10-04 15:23:43 +02:00
for {
if len(tmpPoints) == 0 {
break
}
2021-10-04 15:23:43 +02:00
p := tmpPoints[0]
for k, v := range config.DefTags {
p.AddTag(k, v)
p.SetTime(t)
}
2021-10-04 15:23:43 +02:00
sink.Write(p)
tmpPoints = tmpPoints[1:]
2021-03-25 17:47:08 +01:00
}
}
2021-10-12 13:43:58 +02:00
if err := sink.Flush(); err != nil {
log.Printf("sink error: %s\n", err)
}
if clicfg["once"] == "true" {
2021-11-25 15:11:39 +01:00
shutdown(&wg, config.Collectors, sink, recv, clicfg["pidfile"])
return
}
2021-03-25 14:46:25 +01:00
}
}
}()
// Wait until receiving an interrupt
2021-03-25 14:46:25 +01:00
wg.Wait()
}