From f822f00cdc2a58f86d11770a304ebd0eae93dc32 Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 26 Mar 2021 16:48:09 +0100 Subject: [PATCH] Add sink for InfluxDB (with the original InfluxDB client) --- config.json | 4 ++- go.mod | 5 +++- go.sum | 59 +++++++++++++++++++++++++++++++++++++++++++++ metric-collector.go | 49 ++++++++++++++++++------------------- sinks/influxSink.go | 44 +++++++++++++++++++++++++++++++++ sinks/sink.go | 19 +++++++++++++++ 6 files changed, 153 insertions(+), 27 deletions(-) create mode 100644 sinks/influxSink.go create mode 100644 sinks/sink.go diff --git a/config.json b/config.json index 5340a4e..464b6f6 100644 --- a/config.json +++ b/config.json @@ -3,7 +3,9 @@ "user": "admin", "password": "12345", "host": "localhost", - "port": "8080" + "port": "8080", + "database": "testdb", + "type": "influxdb" }, "interval" : 3, "duration" : 1, diff --git a/go.mod b/go.mod index 9771f8d..90b6fe0 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module github.com/ClusterCockpit/cc-metric-collector go 1.16 -require github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 +require ( + github.com/influxdata/influxdb-client-go/v2 v2.2.2 + github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 +) diff --git a/go.sum b/go.sum index 3a13697..f63eae9 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,61 @@ +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepmap/oapi-codegen v1.3.13 h1:9HKGCsdJqE4dnrQ8VerFS0/1ZOJPmAhN+g8xgp8y3K4= +github.com/deepmap/oapi-codegen v1.3.13/go.mod h1:WAmG5dWY8/PYHt4vKxlt90NsbHMAOCiteYKZMiIRfOo= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/getkin/kin-openapi v0.13.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= +github.com/influxdata/influxdb-client-go v1.4.0 h1:+KavOkwhLClHFfYcJMHHnTL5CZQhXJzOm5IKHI9BqJk= +github.com/influxdata/influxdb-client-go/v2 v2.2.2 h1:O0CGIuIwQafvAxttAJ/VqMKfbWWn2Mt8rbOmaM2Zj4w= +github.com/influxdata/influxdb-client-go/v2 v2.2.2/go.mod h1:fa/d1lAdUHxuc1jedx30ZfNG573oQTQmUni3N6pcW+0= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 h1:vilfsDSy7TDxedi9gyBkMvAirat/oRcL0lFdJBf6tdM= github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343 h1:00ohfJ4K98s3m6BGUoBd8nyfp4Yl0GoIKvw5abItTjI= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/metric-collector.go b/metric-collector.go index ea5c675..b939a97 100644 --- a/metric-collector.go +++ b/metric-collector.go @@ -4,7 +4,7 @@ import ( "encoding/json" "fmt" "github.com/ClusterCockpit/cc-metric-collector/collectors" - protocol "github.com/influxdata/line-protocol" + "github.com/ClusterCockpit/cc-metric-collector/sinks" "log" "os" "os/signal" @@ -24,8 +24,9 @@ var Collectors = map[string]collectors.MetricGetter{ "lustrestat": &collectors.LustreCollector{}, } -// Pointer to the InfluxDB line protocol encoder -var serializer *protocol.Encoder +var Sinks = map[string]sinks.SinkFuncs{ + "influxdb": &sinks.InfluxSink{}, +} // Structure of the configuration file type GlobalConfig struct { @@ -34,6 +35,8 @@ type GlobalConfig struct { Password string `json:"password"` Host string `json:"host"` Port string `json:"port"` + Database string `json:"database"` + Type string `json:"type"` } `json:"sink"` Interval int `json:"interval"` Duration int `json:"duration"` @@ -54,7 +57,7 @@ func LoadConfiguration(file string, config *GlobalConfig) error { // Register an interrupt handler for Ctrl+C and similar. At signal, // all collectors are closed -func shutdown(wg *sync.WaitGroup, config *GlobalConfig) { +func shutdown(wg *sync.WaitGroup, config *GlobalConfig, sink sinks.SinkFuncs) { sigs := make(chan os.Signal, 1) signal.Notify(sigs, os.Interrupt) @@ -67,22 +70,11 @@ func shutdown(wg *sync.WaitGroup, config *GlobalConfig) { col.Close() } time.Sleep(1 * time.Second) + sink.Close() wg.Done() }(wg) } -// Create a new measurement in InfluxDB line protocol -func setupProtocol(scope string, tags map[string]string, fields map[string]interface{}, t time.Time) { - cur, err := protocol.New(scope, tags, fields, t) - if err != nil { - log.Print(err) - } - _, err = serializer.Encode(cur) - if err != nil { - log.Print(err) - } -} - func main() { var config GlobalConfig var wg sync.WaitGroup @@ -117,13 +109,19 @@ func main() { return } } - // Register interrupt handler - shutdown(&wg, &config) + if _, found := Sinks[config.Sink.Type]; !found { + log.Print("Invalid sink type '", config.Sink.Type, "' in configuration") + return + } + // Setup sink + sink := Sinks[config.Sink.Type] + err = sink.Init(config.Sink.Host, config.Sink.Port, config.Sink.User, config.Sink.Password, config.Sink.Database) + if err != nil { + return + } - // Setup InfluxDB line protocol encoder - serializer = protocol.NewEncoder(os.Stdout) - serializer.SetPrecision(time.Second) - serializer.SetMaxLineBytes(1024) + // Register interrupt handler + shutdown(&wg, &config, sink) // Initialize all collectors for _, c := range config.Collectors { @@ -186,20 +184,21 @@ func main() { } } } + // Send out node metrics - setupProtocol("node", map[string]string{"host": host}, nodeFields, t) + sink.Write("node", map[string]string{"host": host}, nodeFields, t) // Send out socket metrics (if any) if scount > 0 { for sid, socket := range socketsFields { - setupProtocol("socket", map[string]string{"socket": fmt.Sprintf("%d", sid), "host": host}, socket, t) + sink.Write("socket", map[string]string{"socket": fmt.Sprintf("%d", sid), "host": host}, socket, t) } } // Send out CPU metrics (if any) if ccount > 0 { for cid, cpu := range cpuFields { - setupProtocol("cpu", map[string]string{"cpu": fmt.Sprintf("%d", cid), "host": host}, cpu, t) + sink.Write("cpu", map[string]string{"cpu": fmt.Sprintf("%d", cid), "host": host}, cpu, t) } } } diff --git a/sinks/influxSink.go b/sinks/influxSink.go new file mode 100644 index 0000000..628a403 --- /dev/null +++ b/sinks/influxSink.go @@ -0,0 +1,44 @@ +package sinks + +import ( + "context" + "fmt" + influxdb2 "github.com/influxdata/influxdb-client-go/v2" + influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api" + "log" + "time" +) + +type InfluxSink struct { + Sink + client influxdb2.Client + writeApi influxdb2Api.WriteAPIBlocking + retPolicy string + organization string +} + +func (s *InfluxSink) Init(host string, port string, user string, password string, database string) error { + s.host = host + s.port = port + s.user = user + s.password = password + s.database = database + s.organization = "" + uri := fmt.Sprintf("http://%s:%s", host, port) + auth := fmt.Sprintf("%s:%s", user, password) + log.Print("Using URI ", uri, " for connection") + s.client = influxdb2.NewClient(uri, auth) + s.writeApi = s.client.WriteAPIBlocking(s.organization, s.database) + return nil +} + +func (s *InfluxSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error { + p := influxdb2.NewPoint(measurement, tags, fields, t) + err := s.writeApi.WritePoint(context.Background(), p) + return err +} + +func (s *InfluxSink) Close() { + log.Print("Closing InfluxDB connection") + s.client.Close() +} diff --git a/sinks/sink.go b/sinks/sink.go new file mode 100644 index 0000000..1efdc81 --- /dev/null +++ b/sinks/sink.go @@ -0,0 +1,19 @@ +package sinks + +import ( + "time" +) + +type Sink struct { + host string + port string + user string + password string + database string +} + +type SinkFuncs interface { + Init(host string, port string, user string, password string, database string) error + Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error + Close() +}