Update NetstatCollector to derive bandwidths and use an include list

collectors/netstatMetric.go

@@ -1,93 +1,138 @@
 package collectors
 
 import (
+	"bufio"
 	"encoding/json"
-	"io/ioutil"
+	"errors"
-	"log"
+	"os"
 	"strconv"
 	"strings"
 	"time"
 
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
 )
 
 const NETSTATFILE = `/proc/net/dev`
 
 type NetstatCollectorConfig struct {
-	ExcludeDevices []string `json:"exclude_devices"`
+	IncludeDevices []string `json:"include_devices"`
+}
+
+type NetstatCollectorMetric struct {
+	index     int
+	lastValue float64
 }
 
 type NetstatCollector struct {
 	metricCollector
 	config        NetstatCollectorConfig
-	matches map[int]string
+	matches       map[string]map[string]NetstatCollectorMetric
+	devtags       map[string]map[string]string
+	lastTimestamp time.Time
 }
 
 func (m *NetstatCollector) Init(config json.RawMessage) error {
 	m.name = "NetstatCollector"
 	m.setup()
+	m.lastTimestamp = time.Now()
 	m.meta = map[string]string{"source": m.name, "group": "Network"}
-	m.matches = map[int]string{
+	m.devtags = make(map[string]map[string]string)
-		1:  "net_bytes_in",
+	nameIndexMap := map[string]int{
-		9:  "net_bytes_out",
+		"net_bytes_in":  1,
-		2:  "net_pkts_in",
+		"net_pkts_in":   2,
-		10: "net_pkts_out",
+		"net_bytes_out": 9,
+		"net_pkts_out":  10,
 	}
+	m.matches = make(map[string]map[string]NetstatCollectorMetric)
 	if len(config) > 0 {
 		err := json.Unmarshal(config, &m.config)
 		if err != nil {
-			log.Print(err.Error())
+			cclog.ComponentError(m.name, "Error reading config:", err.Error())
 			return err
 		}
 	}
-	_, err := ioutil.ReadFile(string(NETSTATFILE))
+	file, err := os.Open(string(NETSTATFILE))
-	if err == nil {
-		m.init = true
-	}
-	return nil
-}
-
-func (m *NetstatCollector) Read(interval time.Duration, output chan lp.CCMetric) {
-	data, err := ioutil.ReadFile(string(NETSTATFILE))
 	if err != nil {
-		log.Print(err.Error())
+		cclog.ComponentError(m.name, err.Error())
-		return
+		return err
 	}
+	defer file.Close()
 
-	lines := strings.Split(string(data), "\n")
+	scanner := bufio.NewScanner(file)
-	for _, l := range lines {
+	for scanner.Scan() {
+		l := scanner.Text()
 		if !strings.Contains(l, ":") {
 			continue
 		}
 		f := strings.Fields(l)
-		dev := f[0][0 : len(f[0])-1]
+		dev := strings.Trim(f[0], ": ")
-		cont := false
+		if _, ok := stringArrayContains(m.config.IncludeDevices, dev); ok {
-		for _, d := range m.config.ExcludeDevices {
+			m.matches[dev] = make(map[string]NetstatCollectorMetric)
-			if d == dev {
+			for name, idx := range nameIndexMap {
-				cont = true
+				m.matches[dev][name] = NetstatCollectorMetric{
+					index:     idx,
+					lastValue: 0,
 				}
 			}
-		if cont {
+			m.devtags[dev] = map[string]string{"device": dev, "type": "node"}
+		}
+	}
+	if len(m.devtags) == 0 {
+		return errors.New("no devices to collector metrics found")
+	}
+	m.init = true
+	return nil
+}
+
+func (m *NetstatCollector) Read(interval time.Duration, output chan lp.CCMetric) {
+	if !m.init {
+		return
+	}
+	now := time.Now()
+	file, err := os.Open(string(NETSTATFILE))
+	if err != nil {
+		cclog.ComponentError(m.name, err.Error())
+		return
+	}
+	defer file.Close()
+	tdiff := now.Sub(m.lastTimestamp)
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		l := scanner.Text()
+		if !strings.Contains(l, ":") {
 			continue
 		}
-		tags := map[string]string{"device": dev, "type": "node"}
+		f := strings.Fields(l)
-		for i, name := range m.matches {
+		dev := strings.Trim(f[0], ":")
-			v, err := strconv.ParseInt(f[i], 10, 0)
+
+		if devmetrics, ok := m.matches[dev]; ok {
+			for name, data := range devmetrics {
+				v, err := strconv.ParseFloat(f[data.index], 64)
 				if err == nil {
-				y, err := lp.New(name, tags, m.meta, map[string]interface{}{"value": int(float64(v) * 1.0e-3)}, time.Now())
+					vdiff := v - data.lastValue
+					value := vdiff / tdiff.Seconds()
+					if data.lastValue == 0 {
+						value = 0
+					}
+					data.lastValue = v
+					y, err := lp.New(name, m.devtags[dev], m.meta, map[string]interface{}{"value": value}, now)
 					if err == nil {
 						switch {
 						case strings.Contains(name, "byte"):
-						y.AddMeta("unit", "Byte")
+							y.AddMeta("unit", "bytes/sec")
 						case strings.Contains(name, "pkt"):
-						y.AddMeta("unit", "Packets")
+							y.AddMeta("unit", "packets/sec")
 						}
 						output <- y
 					}
+					devmetrics[name] = data
 				}
 			}
 		}
-
+	}
+	m.lastTimestamp = time.Now()
 }
 
 func (m *NetstatCollector) Close() {

collectors/netstatMetric.md

@@ -3,19 +3,19 @@
 
 ```json
   "netstat": {
-    "exclude_devices": [
+    "include_devices": [
-      "lo"
+      "eth0"
     ]
   }
 ```
 
-The `netstat` collector reads data from `/proc/net/dev` and outputs a handful **node** metrics. If a device is not required, it can be excluded from forwarding it to the sink. Commonly the `lo` device should be excluded.
+The `netstat` collector reads data from `/proc/net/dev` and outputs a handful **node** metrics. With the `include_devices` list you can specify which network devices should be measured. **Note**: Most other collectors use an _exclude_ list instead of an include list.
 
 Metrics:
-* `bytes_in`
+* `net_bytes_in` (`unit=bytes/sec`)
-* `bytes_out`
+* `net_bytes_out` (`unit=bytes/sec`)
-* `pkts_in`
+* `net_pkts_in` (`unit=packets/sec`)
-* `pkts_out`
+* `net_pkts_out` (`unit=packets/sec`)
 
 The device name is added as tag `device`.