Hand over full config to Sink and Receiver

This commit is contained in:
Thomas Roehl 2021-05-18 15:16:10 +02:00
parent b1e6b8e379
commit 1b9cb8955c
5 changed files with 137 additions and 70 deletions

View File

@ -5,6 +5,7 @@ import (
"flag" "flag"
"fmt" "fmt"
"github.com/ClusterCockpit/cc-metric-collector/collectors" "github.com/ClusterCockpit/cc-metric-collector/collectors"
"github.com/ClusterCockpit/cc-metric-collector/receivers"
"github.com/ClusterCockpit/cc-metric-collector/sinks" "github.com/ClusterCockpit/cc-metric-collector/sinks"
"log" "log"
"os" "os"
@ -34,19 +35,17 @@ var Sinks = map[string]sinks.SinkFuncs{
"nats": &sinks.NatsSink{}, "nats": &sinks.NatsSink{},
} }
var Receivers = map[string]receivers.ReceiverFuncs{
"nats": &receivers.NatsReceiver{},
}
// Structure of the configuration file // Structure of the configuration file
type GlobalConfig struct { type GlobalConfig struct {
Sink struct { Sink sinks.SinkConfig `json:"sink"`
User string `json:"user"`
Password string `json:"password"`
Host string `json:"host"`
Port string `json:"port"`
Database string `json:"database"`
Type string `json:"type"`
} `json:"sink"`
Interval int `json:"interval"` Interval int `json:"interval"`
Duration int `json:"duration"` Duration int `json:"duration"`
Collectors []string `json:"collectors"` Collectors []string `json:"collectors"`
Receiver receivers.ReceiverConfig `json:"receiver"`
} }
// Load JSON configuration file // Load JSON configuration file
@ -91,7 +90,7 @@ func SetLogging(logfile string) error {
// Register an interrupt handler for Ctrl+C and similar. At signal, // Register an interrupt handler for Ctrl+C and similar. At signal,
// all collectors are closed // all collectors are closed
func shutdown(wg *sync.WaitGroup, config *GlobalConfig, sink sinks.SinkFuncs) { func shutdown(wg *sync.WaitGroup, config *GlobalConfig, sink sinks.SinkFuncs, recv receivers.ReceiverFuncs) {
sigs := make(chan os.Signal, 1) sigs := make(chan os.Signal, 1)
signal.Notify(sigs, os.Interrupt) signal.Notify(sigs, os.Interrupt)
@ -104,6 +103,9 @@ func shutdown(wg *sync.WaitGroup, config *GlobalConfig, sink sinks.SinkFuncs) {
col.Close() col.Close()
} }
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
if recv != nil {
recv.Close()
}
sink.Close() sink.Close()
wg.Done() wg.Done()
}(wg) }(wg)
@ -112,6 +114,9 @@ func shutdown(wg *sync.WaitGroup, config *GlobalConfig, sink sinks.SinkFuncs) {
func main() { func main() {
var config GlobalConfig var config GlobalConfig
var wg sync.WaitGroup var wg sync.WaitGroup
var recv receivers.ReceiverFuncs = nil
var use_recv bool
use_recv = false
wg.Add(1) wg.Add(1)
host, err := os.Hostname() host, err := os.Hostname()
if err != nil { if err != nil {
@ -159,13 +164,29 @@ func main() {
} }
// Setup sink // Setup sink
sink := Sinks[config.Sink.Type] sink := Sinks[config.Sink.Type]
err = sink.Init(config.Sink.Host, config.Sink.Port, config.Sink.User, config.Sink.Password, config.Sink.Database) err = sink.Init(config.Sink)
if err != nil { if err != nil {
log.Print(err)
return return
} }
// Setup receiver
if config.Receiver.Type != "none" {
if _, found := Receivers[config.Receiver.Type]; !found {
log.Print("Invalid receiver type '", config.Receiver.Type, "' in configuration")
return
} else {
recv = Receivers[config.Receiver.Type]
err = recv.Init(config.Receiver, sink)
if err == nil {
use_recv = true
} else {
log.Print(err)
}
}
}
// Register interrupt handler // Register interrupt handler
shutdown(&wg, &config, sink) shutdown(&wg, &config, sink, recv)
// Initialize all collectors // Initialize all collectors
tmp := make([]string, 0) tmp := make([]string, 0)
@ -203,6 +224,11 @@ func main() {
cpuFields[s] = make(map[string]interface{}) cpuFields[s] = make(map[string]interface{})
} }
// Start receiver
if use_recv {
recv.Start()
}
go func() { go func() {
for { for {
select { select {

View File

@ -2,6 +2,7 @@ package sinks
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
influxdb2 "github.com/influxdata/influxdb-client-go/v2" influxdb2 "github.com/influxdata/influxdb-client-go/v2"
influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api" influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api"
@ -14,24 +15,39 @@ type InfluxSink struct {
client influxdb2.Client client influxdb2.Client
writeApi influxdb2Api.WriteAPIBlocking writeApi influxdb2Api.WriteAPIBlocking
retPolicy string retPolicy string
organization string
} }
func (s *InfluxSink) Init(host string, port string, user string, password string, database string) error { func (s *InfluxSink) connect() error {
s.host = host var auth string
s.port = port uri := fmt.Sprintf("http://%s:%s", s.host, s.port)
s.user = user if len(s.user) == 0 {
s.password = password auth = s.password
s.database = database } else {
s.organization = "" auth = fmt.Sprintf("%s:%s", s.user, s.password)
uri := fmt.Sprintf("http://%s:%s", host, port) }
auth := fmt.Sprintf("%s:%s", user, password) log.Print("Using URI ", uri, " Org ", s.organization, " Bucket ", s.database)
log.Print("Using URI ", uri, " for connection")
s.client = influxdb2.NewClient(uri, auth) s.client = influxdb2.NewClient(uri, auth)
s.writeApi = s.client.WriteAPIBlocking(s.organization, s.database) s.writeApi = s.client.WriteAPIBlocking(s.organization, s.database)
return nil return nil
} }
func (s *InfluxSink) Init(config SinkConfig) error {
if len(config.Host) == 0 ||
len(config.Port) == 0 ||
len(config.Database) == 0 ||
len(config.Organization) == 0 ||
len(config.Password) == 0 {
return errors.New("Not all configuration variables set required by InfluxSink")
}
s.host = config.Host
s.port = config.Port
s.database = config.Database
s.organization = config.Organization
s.user = config.User
s.password = config.Password
return s.connect()
}
func (s *InfluxSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error { func (s *InfluxSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error {
p := influxdb2.NewPoint(measurement, tags, fields, t) p := influxdb2.NewPoint(measurement, tags, fields, t)
err := s.writeApi.WritePoint(context.Background(), p) err := s.writeApi.WritePoint(context.Background(), p)

View File

@ -2,6 +2,7 @@ package sinks
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
protocol "github.com/influxdata/line-protocol" protocol "github.com/influxdata/line-protocol"
nats "github.com/nats-io/nats.go" nats "github.com/nats-io/nats.go"
@ -16,21 +17,11 @@ type NatsSink struct {
buffer *bytes.Buffer buffer *bytes.Buffer
} }
func (s *NatsSink) Init(host string, port string, user string, password string, database string) error { func (s *NatsSink) connect() error {
s.host = host
s.port = port
s.user = user
s.password = password
s.database = database
// Setup Influx line protocol
s.buffer = &bytes.Buffer{}
s.buffer.Grow(1025)
s.encoder = protocol.NewEncoder(s.buffer)
s.encoder.SetPrecision(time.Second)
s.encoder.SetMaxLineBytes(1024)
// Setup infos for connection
uinfo := nats.UserInfo(s.user, s.password) uinfo := nats.UserInfo(s.user, s.password)
uri := fmt.Sprintf("nats://%s:%s", s.host, s.port) uri := fmt.Sprintf("nats://%s:%s", s.host, s.port)
log.Print("Using URI ", uri)
s.client = nil
nc, err := nats.Connect(uri, uinfo) nc, err := nats.Connect(uri, uinfo)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
@ -40,7 +31,30 @@ func (s *NatsSink) Init(host string, port string, user string, password string,
return nil return nil
} }
func (s *NatsSink) Init(config SinkConfig) error {
if len(config.Host) == 0 ||
len(config.Port) == 0 ||
len(config.Database) == 0 {
return errors.New("Not all configuration variables set required by NatsSink")
}
s.host = config.Host
s.port = config.Port
s.database = config.Database
s.organization = config.Organization
s.user = config.User
s.password = config.Password
// Setup Influx line protocol
s.buffer = &bytes.Buffer{}
s.buffer.Grow(1025)
s.encoder = protocol.NewEncoder(s.buffer)
s.encoder.SetPrecision(time.Second)
s.encoder.SetMaxLineBytes(1024)
// Setup infos for connection
return s.connect()
}
func (s *NatsSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error { func (s *NatsSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error {
if s.client != nil {
m, err := protocol.New(measurement, tags, fields, t) m, err := protocol.New(measurement, tags, fields, t)
if err != nil { if err != nil {
log.Print(err) log.Print(err)
@ -52,9 +66,14 @@ func (s *NatsSink) Write(measurement string, tags map[string]string, fields map[
return err return err
} }
s.client.Publish(s.database, s.buffer.Bytes()) s.client.Publish(s.database, s.buffer.Bytes())
}
return nil return nil
} }
func (s *NatsSink) Close() { func (s *NatsSink) Close() {
log.Print("Closing Nats connection")
if s.client != nil {
s.client.Close() s.client.Close()
}
} }

View File

@ -4,16 +4,27 @@ import (
"time" "time"
) )
type SinkConfig struct {
Host string `json:"host"`
Port string `json:"port"`
Database string `json:"database"`
User string `json:"user"`
Password string `json:"password"`
Organization string `json:"organization"`
Type string `json:"type"`
}
type Sink struct { type Sink struct {
host string host string
port string port string
user string user string
password string password string
database string database string
organization string
} }
type SinkFuncs interface { type SinkFuncs interface {
Init(host string, port string, user string, password string, database string) error Init(config SinkConfig) error
Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error
Close() Close()
} }

View File

@ -11,12 +11,7 @@ type StdoutSink struct {
Sink Sink
} }
func (s *StdoutSink) Init(host string, port string, user string, password string, database string) error { func (s *StdoutSink) Init(config SinkConfig) error {
s.host = host
s.port = port
s.user = user
s.password = password
s.database = database
return nil return nil
} }