Merge branch 'develop' of github.com:ClusterCockpit/cc-metric-collector into develop

sinks/httpSink.go

@@ -45,6 +45,9 @@ type HttpSinkConfig struct {
 
 	// Maximum number of retries to connect to the http server (default: 3)
 	MaxRetries int `json:"max_retries,omitempty"`
+
+	// Timestamp precision
+	Precision string `json:"precision,omitempty"`
 }
 
 type key_value_pair struct {
@@ -141,7 +144,7 @@ func (s *HttpSink) Write(m lp.CCMetric) error {
 
 	// Check that encoding worked
 	if err != nil {
-		return fmt.Errorf("Encoding failed: %v", err)
+		return fmt.Errorf("encoding failed: %v", err)
 	}
 
 	if s.config.flushDelay == 0 {
@@ -268,6 +271,7 @@ func NewHttpSink(name string, config json.RawMessage) (Sink, error) {
 	s.config.Timeout = "5s"
 	s.config.FlushDelay = "5s"
 	s.config.MaxRetries = 3
+	s.config.Precision = "ns"
 	cclog.ComponentDebug(s.name, "Init()")
 
 	// Read config
@@ -315,6 +319,19 @@ func NewHttpSink(name string, config json.RawMessage) (Sink, error) {
 			cclog.ComponentDebug(s.name, "Init(): flushDelay", t)
 		}
 	}
+	precision := influx.Nanosecond
+	if len(s.config.Precision) > 0 {
+		switch s.config.Precision {
+		case "s":
+			precision = influx.Second
+		case "ms":
+			precision = influx.Millisecond
+		case "us":
+			precision = influx.Microsecond
+		case "ns":
+			precision = influx.Nanosecond
+		}
+	}
 
 	// Create http client
 	s.client = &http.Client{
@@ -326,7 +343,7 @@ func NewHttpSink(name string, config json.RawMessage) (Sink, error) {
 	}
 
 	// Configure influx line protocol encoder
-	s.encoder.SetPrecision(influx.Nanosecond)
+	s.encoder.SetPrecision(precision)
 	s.extended_tag_list = make([]key_value_pair, 0)
 
 	return s, nil

sinks/httpSink.md

@@ -18,7 +18,8 @@ The `http` sink uses POST requests to a HTTP server to submit the metrics in the
     "timeout": "5s",
     "idle_connection_timeout" : "5s",
     "flush_delay": "2s",
-    "batch_size": 1000
+    "batch_size": 1000,
+    "precision": "s"
   }
 }
 ```
@@ -34,3 +35,8 @@ The `http` sink uses POST requests to a HTTP server to submit the metrics in the
 - `idle_connection_timeout`: Timeout for idle connections (default '120s'). Should be larger than the measurement interval to keep the connection open
 - `flush_delay`: Batch all writes arriving in during this duration (default '1s', batching can be disabled by setting it to 0)
 - `batch_size`: Maximal batch size. If `batch_size` is reached before the end of `flush_delay`, the metrics are sent without further delay
+- `precision`: Precision of the timestamp. Valid values are 's', 'ms', 'us' and 'ns'. (default is 'ns')
+
+### Using HttpSink for communication with cc-metric-store
+
+The cc-metric-store only accepts metrics with a timestamp precision in seconds, so it is required to set `"precision": "s"`.

sinks/influxAsyncSink.md

@@ -9,7 +9,6 @@ The `influxasync` sink uses the official [InfluxDB golang client](https://pkg.go
 {
   "<name>": {
     "type": "influxasync",
-    "meta_as_tags" : true,
     "database" : "mymetrics",
     "host": "dbhost.example.com",
     "port": "4222",
@@ -21,13 +20,13 @@ The `influxasync` sink uses the official [InfluxDB golang client](https://pkg.go
     "retry_interval" : "1s",
     "retry_exponential_base" : 2,
     "max_retries": 20,
-    "max_retry_time" : "168h"
+    "max_retry_time" : "168h",
+    "meta_as_tags" : [],
   }
 }
 ```
 
 - `type`: makes the sink an `influxdb` sink
-- `meta_as_tags`: print all meta information as tags in the output (optional)
 - `database`: All metrics are written to this bucket 
 - `host`: Hostname of the InfluxDB database server
 - `port`: Portnumber (as string) of the InfluxDB database server
@@ -40,5 +39,6 @@ The `influxasync` sink uses the official [InfluxDB golang client](https://pkg.go
 - `retry_exponential_base`: The retry interval is exponentially increased with this base, default 2
 - `max_retries`: Maximal number of retry attempts
 - `max_retry_time`: Maximal time to retry failed writes, default 168h (one week)
+- `meta_as_tags`: move meta information keys to tags (optional)
 
-For information about the calculation of the retry interval settings, see [offical influxdb-client-go documentation](https://github.com/influxdata/influxdb-client-go#handling-of-failed-async-writes)
+For information about the calculation of the retry interval settings, see [offical influxdb-client-go documentation](https://github.com/influxdata/influxdb-client-go#handling-of-failed-async-writes)

sinks/influxSink.md

@@ -8,7 +8,6 @@ The `influxdb` sink uses the official [InfluxDB golang client](https://pkg.go.de
 {
   "<name>": {
     "type": "influxdb",
-    "meta_as_tags" : true,
     "database" : "mymetrics",
     "host": "dbhost.example.com",
     "port": "4222",
@@ -19,6 +18,7 @@ The `influxdb` sink uses the official [InfluxDB golang client](https://pkg.go.de
     "flush_delay" : "1s",
     "batch_size" : 1000,
     "use_gzip": true
+    "meta_as_tags" : [],
   }
 }
 ```
@@ -36,7 +36,9 @@ The `influxdb` sink uses the official [InfluxDB golang client](https://pkg.go.de
 - `batch_size`: Maximal batch size. If `batch_size` is reached before the end of `flush_delay`, the metrics are sent without further delay
 
 Influx client options:
-
+=======
+- `batch_size`: Maximal batch size
+- `meta_as_tags`: move meta information keys to tags (optional)
 - `http_request_timeout`: HTTP request timeout
 - `retry_interval`: retry interval
 - `max_retry_interval`: maximum delay between each retry attempt

sinks/natsSink.md

@@ -8,20 +8,21 @@ The `nats` sink publishes all metrics into a NATS network. The publishing key is
 {
   "<name>": {
     "type": "nats",
-    "meta_as_tags" : true,
     "database" : "mymetrics",
     "host": "dbhost.example.com",
     "port": "4222",
     "user": "exampleuser",
-    "password" : "examplepw"
+    "password" : "examplepw",
+    "meta_as_tags" : [],
   }
 }
 ```
 
 - `type`: makes the sink an `nats` sink
-- `meta_as_tags`: print all meta information as tags in the output (optional)
 - `database`: All metrics are published with this subject
 - `host`: Hostname of the NATS server
 - `port`: Port number (as string) of the NATS server
 - `user`: Username for basic authentication
 - `password`: Password for basic authentication
+- `meta_as_tags`: print all meta information as tags in the output (optional)
+

sinks/stdoutSink.md

@@ -9,14 +9,14 @@ The `stdout` sink is the most simple sink provided by cc-metric-collector. It wr
 {
   "<name>": {
     "type": "stdout",
-    "meta_as_tags" : true,
+    "meta_as_tags" : [],
     "output_file" : "mylogfile.log"
   }
 }
 ```
 
 - `type`: makes the sink an `stdout` sink
-- `meta_as_tags`: print all meta information as tags in the output (optional)
+- `meta_as_tags`: print meta information as tags in the output (optional)
 - `output_file`: Write all data to the selected file (optional). There are two 'special' files: `stdout` and `stderr`. If this option is not provided, the default value is `stdout`