cc-metric-collector/sinks/influxSink.go

package sinks

import (
	"bytes"
	"context"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"sync"
	"time"

	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
	influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api"
	influx "github.com/influxdata/line-protocol/v2/lineprotocol"
	"golang.org/x/exp/slices"
)

type InfluxSink struct {
	sink
	client   influxdb2.Client
	writeApi influxdb2Api.WriteAPIBlocking
	config   struct {
		defaultSinkConfig
		Host         string `json:"host,omitempty"`
		Port         string `json:"port,omitempty"`
		Database     string `json:"database,omitempty"`
		User         string `json:"user,omitempty"`
		Password     string `json:"password,omitempty"`
		Organization string `json:"organization,omitempty"`
		SSL          bool   `json:"ssl,omitempty"`
		// Maximum number of points sent to server in single request.
		// Default: 1000
		BatchSize int `json:"batch_size,omitempty"`

		// Time interval for delayed sending of metrics.
		// If the buffers are already filled before the end of this interval,
		// the metrics are sent without further delay.
		// Default: 1s
		FlushInterval string `json:"flush_delay,omitempty"`
		flushDelay    time.Duration

		// Influx client options:

		// HTTP request timeout
		HTTPRequestTimeout string `json:"http_request_timeout"`
		// Retry interval
		InfluxRetryInterval string `json:"retry_interval,omitempty"`
		// maximum delay between each retry attempt
		InfluxMaxRetryInterval string `json:"max_retry_interval,omitempty"`
		// base for the exponential retry delay
		InfluxExponentialBase uint `json:"retry_exponential_base,omitempty"`
		// maximum count of retry attempts of failed writes
		InfluxMaxRetries uint `json:"max_retries,omitempty"`
		// maximum total retry timeout
		InfluxMaxRetryTime string `json:"max_retry_time,omitempty"`
		// Specify whether to use GZip compression in write requests
		InfluxUseGzip bool `json:"use_gzip"`
	}

	// influx line protocol encoder
	encoder influx.Encoder
	// number of records stored in the encoder
	numRecordsInEncoder int
	// List of tags and meta data tags which should be used as tags
	extended_tag_list []key_value_pair
	// Flush() runs in another goroutine and accesses the influx line protocol encoder,
	// so this encoderLock has to protect the encoder and numRecordsInEncoder
	encoderLock sync.Mutex

	// timer to run Flush()
	flushTimer *time.Timer
	// Lock to assure that only one timer is running at a time
	timerLock sync.Mutex

	// WaitGroup to ensure only one send operation is running at a time
	sendWaitGroup sync.WaitGroup
}

// connect connects to the InfluxDB server
func (s *InfluxSink) connect() error {

	// URI options:
	// * http://host:port
	// * https://host:port
	var uri string
	if s.config.SSL {
		uri = fmt.Sprintf("https://%s:%s", s.config.Host, s.config.Port)
	} else {
		uri = fmt.Sprintf("http://%s:%s", s.config.Host, s.config.Port)
	}

	// Authentication options:
	// * token
	// * username:password
	var auth string
	if len(s.config.User) == 0 {
		auth = s.config.Password
	} else {
		auth = fmt.Sprintf("%s:%s", s.config.User, s.config.Password)
	}
	cclog.ComponentDebug(s.name, "connect():",
		"Using URI='"+uri+"'",
		"Org='"+s.config.Organization+"'",
		"Bucket='"+s.config.Database+"'")

	// Set influxDB client options
	clientOptions := influxdb2.DefaultOptions()

	// set HTTP request timeout
	if len(s.config.HTTPRequestTimeout) > 0 {
		if t, err := time.ParseDuration(s.config.HTTPRequestTimeout); err == nil {
			httpRequestTimeout := uint(t.Seconds())
			clientOptions.SetHTTPRequestTimeout(httpRequestTimeout)
		} else {
			cclog.ComponentError(s.name, "connect():", "Failed to parse duration for HTTP RequestTimeout: ", s.config.HTTPRequestTimeout)
		}
	}
	cclog.ComponentDebug(
		s.name,
		"connect():",
		"Influx client options HTTPRequestTimeout:",
		time.Second*time.Duration(clientOptions.HTTPRequestTimeout()))

	// Set retry interval
	if len(s.config.InfluxRetryInterval) > 0 {
		if t, err := time.ParseDuration(s.config.InfluxRetryInterval); err == nil {
			influxRetryInterval := uint(t.Milliseconds())
			clientOptions.SetRetryInterval(influxRetryInterval)
		} else {
			cclog.ComponentError(s.name, "connect():", "Failed to parse duration for Influx RetryInterval: ", s.config.InfluxRetryInterval)
		}
	}
	cclog.ComponentDebug(
		s.name,
		"connect():",
		"Influx client options RetryInterval:",
		time.Millisecond*time.Duration(clientOptions.RetryInterval()))

	// Set the maximum delay between each retry attempt
	if len(s.config.InfluxMaxRetryInterval) > 0 {
		if t, err := time.ParseDuration(s.config.InfluxMaxRetryInterval); err == nil {
			influxMaxRetryInterval := uint(t.Milliseconds())
			clientOptions.SetMaxRetryInterval(influxMaxRetryInterval)
		} else {
			cclog.ComponentError(s.name, "connect():", "Failed to parse duration for Influx MaxRetryInterval: ", s.config.InfluxMaxRetryInterval)
		}
	}
	cclog.ComponentDebug(
		s.name,
		"connect():",
		"Influx client options MaxRetryInterval:",
		time.Millisecond*time.Duration(clientOptions.MaxRetryInterval()))

	// Set the base for the exponential retry delay
	if s.config.InfluxExponentialBase != 0 {
		clientOptions.SetExponentialBase(s.config.InfluxExponentialBase)
	}
	cclog.ComponentDebug(
		s.name,
		"connect():",
		"Influx client options ExponentialBase:",
		clientOptions.ExponentialBase())

	// Set maximum count of retry attempts of failed writes
	if s.config.InfluxMaxRetries != 0 {
		clientOptions.SetMaxRetries(s.config.InfluxMaxRetries)
	}
	cclog.ComponentDebug(
		s.name,
		"connect():",
		"Influx client options MaxRetries:",
		clientOptions.MaxRetries())

	// Set the maximum total retry timeout
	if len(s.config.InfluxMaxRetryTime) > 0 {
		if t, err := time.ParseDuration(s.config.InfluxMaxRetryTime); err == nil {
			influxMaxRetryTime := uint(t.Milliseconds())
			cclog.ComponentDebug(s.name, "connect():", "MaxRetryTime", s.config.InfluxMaxRetryTime)
			clientOptions.SetMaxRetryTime(influxMaxRetryTime)
		} else {
			cclog.ComponentError(s.name, "connect():", "Failed to parse duration for Influx MaxRetryInterval: ", s.config.InfluxMaxRetryInterval)
		}
	}
	cclog.ComponentDebug(
		s.name,
		"connect():",
		"Influx client options MaxRetryTime:",
		time.Millisecond*time.Duration(clientOptions.MaxRetryTime()))

	// Specify whether to use GZip compression in write requests
	clientOptions.SetUseGZip(s.config.InfluxUseGzip)
	cclog.ComponentDebug(
		s.name,
		"connect():",
		"Influx client options UseGZip:",
		clientOptions.UseGZip())

	// Do not check InfluxDB certificate
	clientOptions.SetTLSConfig(
		&tls.Config{
			InsecureSkipVerify: true,
		},
	)

	// Set time precision
	clientOptions.SetPrecision(time.Nanosecond)

	// Create new writeAPI
	s.client = influxdb2.NewClientWithOptions(uri, auth, clientOptions)
	s.writeApi = s.client.WriteAPIBlocking(s.config.Organization, s.config.Database)

	// Check InfluxDB server accessibility
	ok, err := s.client.Ping(context.Background())
	if err != nil {
		return err
	}
	if !ok {
		return fmt.Errorf("connection to %s not healthy", uri)
	}
	return nil
}

// Write sends metric m in influxDB line protocol
func (s *InfluxSink) Write(m lp.CCMetric) error {

	// Lock for encoder usage
	s.encoderLock.Lock()

	// Encode measurement name
	s.encoder.StartLine(m.Name())

	// copy tags and meta data which should be used as tags
	s.extended_tag_list = s.extended_tag_list[:0]
	for key, value := range m.Tags() {
		s.extended_tag_list =
			append(
				s.extended_tag_list,
				key_value_pair{
					key:   key,
					value: value,
				},
			)
	}
	for _, key := range s.config.MetaAsTags {
		if value, ok := m.GetMeta(key); ok {
			s.extended_tag_list =
				append(
					s.extended_tag_list,
					key_value_pair{
						key:   key,
						value: value,
					},
				)
		}
	}

	// Encode tags (they musts be in lexical order)
	slices.SortFunc(
		s.extended_tag_list,
		func(a key_value_pair, b key_value_pair) int {
			if a.key < b.key {
				return -1
			}
			if a.key > b.key {
				return +1
			}
			return 0
		},
	)
	for i := range s.extended_tag_list {
		s.encoder.AddTag(
			s.extended_tag_list[i].key,
			s.extended_tag_list[i].value,
		)
	}

	// Encode fields
	for key, value := range m.Fields() {
		s.encoder.AddField(key, influx.MustNewValue(value))
	}

	// Encode time stamp
	s.encoder.EndLine(m.Time())

	// Check for encoder errors
	if err := s.encoder.Err(); err != nil {
		// Unlock encoder usage
		s.encoderLock.Unlock()

		return fmt.Errorf("Encoding failed: %v", err)
	}
	s.numRecordsInEncoder++

	if s.config.flushDelay == 0 {
		// Unlock encoder usage
		s.encoderLock.Unlock()

		// Directly flush if no flush delay is configured
		return s.Flush()
	} else if s.numRecordsInEncoder == s.config.BatchSize {
		// Unlock encoder usage
		s.encoderLock.Unlock()

		// Stop flush timer
		if s.flushTimer != nil {
			if ok := s.flushTimer.Stop(); ok {
				cclog.ComponentDebug(s.name, "Write(): Stopped flush timer. Batch size limit reached before flush delay")
				s.timerLock.Unlock()
			}
		}

		// Flush if batch size is reached
		return s.Flush()
	} else if s.timerLock.TryLock() {

		// Setup flush timer when flush delay is configured
		// and no other timer is already running
		if s.flushTimer != nil {

			// Restarting existing flush timer
			cclog.ComponentDebug(s.name, "Write(): Restarting flush timer")
			s.flushTimer.Reset(s.config.flushDelay)
		} else {

			// Creating and starting flush timer
			cclog.ComponentDebug(s.name, "Write(): Starting new flush timer")
			s.flushTimer = time.AfterFunc(
				s.config.flushDelay,
				func() {
					defer s.timerLock.Unlock()
					cclog.ComponentDebug(s.name, "Starting flush triggered by flush timer")
					if err := s.Flush(); err != nil {
						cclog.ComponentError(s.name, "Flush triggered by flush timer: flush failed:", err)
					}
				})
		}
	}

	// Unlock encoder usage
	s.encoderLock.Unlock()
	return nil
}

// Flush sends all metrics stored in encoder to InfluxDB server
func (s *InfluxSink) Flush() error {

	// Lock for encoder usage
	// Own lock for as short as possible: the time it takes to clone the buffer.
	s.encoderLock.Lock()

	buf := slices.Clone(s.encoder.Bytes())
	numRecordsInBuf := s.numRecordsInEncoder
	s.encoder.Reset()
	s.numRecordsInEncoder = 0

	// Unlock encoder usage
	s.encoderLock.Unlock()

	if len(buf) == 0 {
		return nil
	}

	cclog.ComponentDebug(s.name, "Flush(): Flushing", numRecordsInBuf, "metrics")

	// Asynchron send of encoder metrics
	s.sendWaitGroup.Add(1)
	go func() {
		defer s.sendWaitGroup.Done()
		startTime := time.Now()
		err := s.writeApi.WriteRecord(context.Background(), string(buf))
		if err != nil {
			cclog.ComponentError(
				s.name,
				"Flush():",
				"Flush failed:", err,
				"(number of records =", numRecordsInBuf,
				", buffer size =", len(buf),
				", send duration =", time.Since(startTime),
				")",
			)
			return
		}
	}()

	return nil
}

func (s *InfluxSink) Close() {
	cclog.ComponentDebug(s.name, "Closing InfluxDB connection")

	// Stop existing timer and immediately flush
	if s.flushTimer != nil {
		if ok := s.flushTimer.Stop(); ok {
			s.timerLock.Unlock()
		}
	}

	// Flush
	if err := s.Flush(); err != nil {
		cclog.ComponentError(s.name, "Close():", "Flush failed:", err)
	}

	// Wait for send operations to finish
	s.sendWaitGroup.Wait()

	s.client.Close()
}

// NewInfluxSink create a new InfluxDB sink
func NewInfluxSink(name string, config json.RawMessage) (Sink, error) {
	s := new(InfluxSink)
	s.name = fmt.Sprintf("InfluxSink(%s)", name)

	// Set config default values
	s.config.BatchSize = 1000
	s.config.FlushInterval = "1s"

	// Read config
	if len(config) > 0 {
		d := json.NewDecoder(bytes.NewReader(config))
		d.DisallowUnknownFields()
		if err := d.Decode(&s.config); err != nil {
			cclog.ComponentError(s.name, "Error reading config:", err.Error())
			return nil, err
		}
	}

	if len(s.config.Host) == 0 {
		return s, errors.New("missing host configuration required by InfluxSink")
	}
	if len(s.config.Port) == 0 {
		return s, errors.New("missing port configuration required by InfluxSink")
	}
	if len(s.config.Database) == 0 {
		return s, errors.New("missing database configuration required by InfluxSink")
	}
	if len(s.config.Organization) == 0 {
		return s, errors.New("missing organization configuration required by InfluxSink")
	}
	if len(s.config.Password) == 0 {
		return s, errors.New("missing password configuration required by InfluxSink")
	}

	// Create lookup map to use meta infos as tags in the output metric
	s.meta_as_tags = make(map[string]bool)
	for _, k := range s.config.MetaAsTags {
		s.meta_as_tags[k] = true
	}

	// Configure flush delay duration
	if len(s.config.FlushInterval) > 0 {
		t, err := time.ParseDuration(s.config.FlushInterval)
		if err == nil {
			s.config.flushDelay = t
		}
	}

	if !(s.config.BatchSize > 0) {
		return s, fmt.Errorf("batch_size=%d in InfluxDB config must be > 0", s.config.BatchSize)
	}

	// Connect to InfluxDB server
	if err := s.connect(); err != nil {
		return s, fmt.Errorf("unable to connect: %v", err)
	}

	// Configure influx line protocol encoder
	s.encoder.SetPrecision(influx.Nanosecond)
	s.extended_tag_list = make([]key_value_pair, 0)

	return s, nil
}