mirror of
https://github.com/ClusterCockpit/cc-metric-collector.git
synced 2024-12-25 15:09:05 +01:00
Add sink for InfluxDB (with the original InfluxDB client)
This commit is contained in:
Thomas Roehl
parent
e92f54b411
commit
f822f00cdc
@ -3,7 +3,9 @@
|
||||
"user": "admin",
|
||||
"password": "12345",
|
||||
"host": "localhost",
|
||||
"port": "8080"
|
||||
"port": "8080",
|
||||
"database": "testdb",
|
||||
"type": "influxdb"
|
||||
},
|
||||
"interval" : 3,
|
||||
"duration" : 1,
|
||||
|
||||
5
go.mod
5
go.mod
@ -2,4 +2,7 @@ module github.com/ClusterCockpit/cc-metric-collector
|
||||
|
||||
go 1.16
|
||||
|
||||
require github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097
|
||||
require (
|
||||
github.com/influxdata/influxdb-client-go/v2 v2.2.2
|
||||
github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097
|
||||
)
|
||||
|
||||
59
go.sum
59
go.sum
@ -1,2 +1,61 @@
|
||||
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/deepmap/oapi-codegen v1.3.13 h1:9HKGCsdJqE4dnrQ8VerFS0/1ZOJPmAhN+g8xgp8y3K4=
|
||||
github.com/deepmap/oapi-codegen v1.3.13/go.mod h1:WAmG5dWY8/PYHt4vKxlt90NsbHMAOCiteYKZMiIRfOo=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/getkin/kin-openapi v0.13.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
|
||||
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
|
||||
github.com/influxdata/influxdb-client-go v1.4.0 h1:+KavOkwhLClHFfYcJMHHnTL5CZQhXJzOm5IKHI9BqJk=
|
||||
github.com/influxdata/influxdb-client-go/v2 v2.2.2 h1:O0CGIuIwQafvAxttAJ/VqMKfbWWn2Mt8rbOmaM2Zj4w=
|
||||
github.com/influxdata/influxdb-client-go/v2 v2.2.2/go.mod h1:fa/d1lAdUHxuc1jedx30ZfNG573oQTQmUni3N6pcW+0=
|
||||
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
|
||||
github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 h1:vilfsDSy7TDxedi9gyBkMvAirat/oRcL0lFdJBf6tdM=
|
||||
github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
|
||||
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
|
||||
github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
|
||||
github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191112182307-2180aed22343 h1:00ohfJ4K98s3m6BGUoBd8nyfp4Yl0GoIKvw5abItTjI=
|
||||
golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
||||
@ -4,7 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/ClusterCockpit/cc-metric-collector/collectors"
|
||||
protocol "github.com/influxdata/line-protocol"
|
||||
"github.com/ClusterCockpit/cc-metric-collector/sinks"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
@ -24,8 +24,9 @@ var Collectors = map[string]collectors.MetricGetter{
|
||||
"lustrestat": &collectors.LustreCollector{},
|
||||
}
|
||||
|
||||
// Pointer to the InfluxDB line protocol encoder
|
||||
var serializer *protocol.Encoder
|
||||
var Sinks = map[string]sinks.SinkFuncs{
|
||||
"influxdb": &sinks.InfluxSink{},
|
||||
}
|
||||
|
||||
// Structure of the configuration file
|
||||
type GlobalConfig struct {
|
||||
@ -34,6 +35,8 @@ type GlobalConfig struct {
|
||||
Password string `json:"password"`
|
||||
Host string `json:"host"`
|
||||
Port string `json:"port"`
|
||||
Database string `json:"database"`
|
||||
Type string `json:"type"`
|
||||
} `json:"sink"`
|
||||
Interval int `json:"interval"`
|
||||
Duration int `json:"duration"`
|
||||
@ -54,7 +57,7 @@ func LoadConfiguration(file string, config *GlobalConfig) error {
|
||||
|
||||
// Register an interrupt handler for Ctrl+C and similar. At signal,
|
||||
// all collectors are closed
|
||||
func shutdown(wg *sync.WaitGroup, config *GlobalConfig) {
|
||||
func shutdown(wg *sync.WaitGroup, config *GlobalConfig, sink sinks.SinkFuncs) {
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, os.Interrupt)
|
||||
|
||||
@ -67,22 +70,11 @@ func shutdown(wg *sync.WaitGroup, config *GlobalConfig) {
|
||||
col.Close()
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
sink.Close()
|
||||
wg.Done()
|
||||
}(wg)
|
||||
}
|
||||
|
||||
// Create a new measurement in InfluxDB line protocol
|
||||
func setupProtocol(scope string, tags map[string]string, fields map[string]interface{}, t time.Time) {
|
||||
cur, err := protocol.New(scope, tags, fields, t)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
}
|
||||
_, err = serializer.Encode(cur)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
var config GlobalConfig
|
||||
var wg sync.WaitGroup
|
||||
@ -117,13 +109,19 @@ func main() {
|
||||
return
|
||||
}
|
||||
}
|
||||
// Register interrupt handler
|
||||
shutdown(&wg, &config)
|
||||
if _, found := Sinks[config.Sink.Type]; !found {
|
||||
log.Print("Invalid sink type '", config.Sink.Type, "' in configuration")
|
||||
return
|
||||
}
|
||||
// Setup sink
|
||||
sink := Sinks[config.Sink.Type]
|
||||
err = sink.Init(config.Sink.Host, config.Sink.Port, config.Sink.User, config.Sink.Password, config.Sink.Database)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Setup InfluxDB line protocol encoder
|
||||
serializer = protocol.NewEncoder(os.Stdout)
|
||||
serializer.SetPrecision(time.Second)
|
||||
serializer.SetMaxLineBytes(1024)
|
||||
// Register interrupt handler
|
||||
shutdown(&wg, &config, sink)
|
||||
|
||||
// Initialize all collectors
|
||||
for _, c := range config.Collectors {
|
||||
@ -186,20 +184,21 @@ func main() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Send out node metrics
|
||||
setupProtocol("node", map[string]string{"host": host}, nodeFields, t)
|
||||
sink.Write("node", map[string]string{"host": host}, nodeFields, t)
|
||||
|
||||
// Send out socket metrics (if any)
|
||||
if scount > 0 {
|
||||
for sid, socket := range socketsFields {
|
||||
setupProtocol("socket", map[string]string{"socket": fmt.Sprintf("%d", sid), "host": host}, socket, t)
|
||||
sink.Write("socket", map[string]string{"socket": fmt.Sprintf("%d", sid), "host": host}, socket, t)
|
||||
}
|
||||
}
|
||||
|
||||
// Send out CPU metrics (if any)
|
||||
if ccount > 0 {
|
||||
for cid, cpu := range cpuFields {
|
||||
setupProtocol("cpu", map[string]string{"cpu": fmt.Sprintf("%d", cid), "host": host}, cpu, t)
|
||||
sink.Write("cpu", map[string]string{"cpu": fmt.Sprintf("%d", cid), "host": host}, cpu, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
44
sinks/influxSink.go
Normal file
44
sinks/influxSink.go
Normal file
@ -0,0 +1,44 @@
|
||||
package sinks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
|
||||
influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
type InfluxSink struct {
|
||||
Sink
|
||||
client influxdb2.Client
|
||||
writeApi influxdb2Api.WriteAPIBlocking
|
||||
retPolicy string
|
||||
organization string
|
||||
}
|
||||
|
||||
func (s *InfluxSink) Init(host string, port string, user string, password string, database string) error {
|
||||
s.host = host
|
||||
s.port = port
|
||||
s.user = user
|
||||
s.password = password
|
||||
s.database = database
|
||||
s.organization = ""
|
||||
uri := fmt.Sprintf("http://%s:%s", host, port)
|
||||
auth := fmt.Sprintf("%s:%s", user, password)
|
||||
log.Print("Using URI ", uri, " for connection")
|
||||
s.client = influxdb2.NewClient(uri, auth)
|
||||
s.writeApi = s.client.WriteAPIBlocking(s.organization, s.database)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *InfluxSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error {
|
||||
p := influxdb2.NewPoint(measurement, tags, fields, t)
|
||||
err := s.writeApi.WritePoint(context.Background(), p)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *InfluxSink) Close() {
|
||||
log.Print("Closing InfluxDB connection")
|
||||
s.client.Close()
|
||||
}
|
||||
19
sinks/sink.go
Normal file
19
sinks/sink.go
Normal file
@ -0,0 +1,19 @@
|
||||
package sinks
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type Sink struct {
|
||||
host string
|
||||
port string
|
||||
user string
|
||||
password string
|
||||
database string
|
||||
}
|
||||
|
||||
type SinkFuncs interface {
|
||||
Init(host string, port string, user string, password string, database string) error
|
||||
Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error
|
||||
Close()
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user