Change storage format

This commit is contained in:
Thomas Roehl
2021-10-04 15:23:43 +02:00
parent 8956ce7ad5
commit 558bbaba59
19 changed files with 874 additions and 283 deletions

View File

@@ -6,8 +6,8 @@ import (
"fmt"
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api"
lp "github.com/influxdata/line-protocol"
"log"
"time"
)
type InfluxSink struct {
@@ -48,8 +48,16 @@ func (s *InfluxSink) Init(config SinkConfig) error {
return s.connect()
}
func (s *InfluxSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error {
p := influxdb2.NewPoint(measurement, tags, fields, t)
func (s *InfluxSink) Write(point lp.MutableMetric) error {
var tags map[string]string
var fields map[string]interface{}
for _, t := range point.TagList() {
tags[t.Key] = t.Value
}
for _, f := range point.FieldList() {
fields[f.Key] = f.Value
}
p := influxdb2.NewPoint(point.Name(), tags, fields, point.Time())
err := s.writeApi.WritePoint(context.Background(), p)
return err
}

View File

@@ -1,7 +1,8 @@
package sinks
import (
"time"
// "time"
lp "github.com/influxdata/line-protocol"
)
type SinkConfig struct {
@@ -25,6 +26,6 @@ type Sink struct {
type SinkFuncs interface {
Init(config SinkConfig) error
Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error
Write(point lp.MutableMetric) error
Close()
}

View File

@@ -4,7 +4,7 @@ import (
"bytes"
"errors"
"fmt"
protocol "github.com/influxdata/line-protocol"
lp "github.com/influxdata/line-protocol"
nats "github.com/nats-io/nats.go"
"log"
"time"
@@ -13,7 +13,7 @@ import (
type NatsSink struct {
Sink
client *nats.Conn
encoder *protocol.Encoder
encoder *lp.Encoder
buffer *bytes.Buffer
}
@@ -46,21 +46,29 @@ func (s *NatsSink) Init(config SinkConfig) error {
// Setup Influx line protocol
s.buffer = &bytes.Buffer{}
s.buffer.Grow(1025)
s.encoder = protocol.NewEncoder(s.buffer)
s.encoder = lp.NewEncoder(s.buffer)
s.encoder.SetPrecision(time.Second)
s.encoder.SetMaxLineBytes(1024)
// Setup infos for connection
return s.connect()
}
func (s *NatsSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error {
func (s *NatsSink) Write(point lp.MutableMetric) error {
if s.client != nil {
m, err := protocol.New(measurement, tags, fields, t)
if err != nil {
log.Print(err)
return err
}
_, err = s.encoder.Encode(m)
// var tags map[string]string
// var fields map[string]interface{}
// for _, t := range point.TagList() {
// tags[t.Key] = t.Value
// }
// for _, f := range point.FieldList() {
// fields[f.Key] = f.Value
// }
// m, err := protocol.New(point.Name(), tags, fields, point.Time())
// if err != nil {
// log.Print(err)
// return err
// }
_, err := s.encoder.Encode(point)
if err != nil {
log.Print(err)
return err

View File

@@ -4,7 +4,8 @@ import (
"fmt"
"math"
"strings"
"time"
// "time"
lp "github.com/influxdata/line-protocol"
)
type StdoutSink struct {
@@ -15,26 +16,40 @@ func (s *StdoutSink) Init(config SinkConfig) error {
return nil
}
func (s *StdoutSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error {
func (s *StdoutSink) Write(point lp.MutableMetric) error {
var tagsstr []string
var fieldstr []string
for k, v := range tags {
tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", k, v))
for _, t := range point.TagList() {
tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", t.Key, t.Value))
}
for k, v := range fields {
switch v.(type) {
for _, f := range point.FieldList() {
switch f.Value.(type) {
case float64:
if !math.IsNaN(v.(float64)) {
fieldstr = append(fieldstr, fmt.Sprintf("%s=%v", k, v.(float64)))
if !math.IsNaN(f.Value.(float64)) {
fieldstr = append(fieldstr, fmt.Sprintf("%s=%v", f.Key, f.Value.(float64)))
} else {
fieldstr = append(fieldstr, fmt.Sprintf("%s=0.0", f.Key))
}
case float32:
if !math.IsNaN(float64(f.Value.(float32))) {
fieldstr = append(fieldstr, fmt.Sprintf("%s=%v", f.Key, f.Value.(float32)))
} else {
fieldstr = append(fieldstr, fmt.Sprintf("%s=0.0", f.Key))
}
case int:
fieldstr = append(fieldstr, fmt.Sprintf("%s=%d", f.Key, f.Value.(int)))
case int64:
fieldstr = append(fieldstr, fmt.Sprintf("%s=%d", f.Key, f.Value.(int64)))
case string:
fieldstr = append(fieldstr, fmt.Sprintf("%s=%q", k, v.(string)))
fieldstr = append(fieldstr, fmt.Sprintf("%s=%q", f.Key, f.Value.(string)))
default:
fieldstr = append(fieldstr, fmt.Sprintf("%s=%v", f.Key, f.Value))
}
}
if len(tagsstr) > 0 {
fmt.Printf("%s,%s %s %d\n", measurement, strings.Join(tagsstr, ","), strings.Join(fieldstr, ","), t.Unix())
fmt.Printf("%s,%s %s %d\n", point.Name(), strings.Join(tagsstr, ","), strings.Join(fieldstr, ","), point.Time().Unix())
} else {
fmt.Printf("%s %s %d\n", measurement, strings.Join(fieldstr, ","), t.Unix())
fmt.Printf("%s %s %d\n", point.Name(), strings.Join(fieldstr, ","), point.Time().Unix())
}
return nil
}