Split diskstat Collector (#38)

* Split diskstats (free, total space) and iostats (reads, writes, ...

* Add iostat Collector to CollectorManager

collectors/README.md

@@ -18,6 +18,7 @@ In contrast to the configuration files for sinks and receivers, the collectors c
 
 * [`cpustat`](./cpustatMetric.md)
 * [`memstat`](./memstatMetric.md)
+* [`iostat`](./iostatMetric.md)
 * [`diskstat`](./diskstatMetric.md)
 * [`loadavg`](./loadavgMetric.md)
 * [`netstat`](./netstatMetric.md)

collectors/collectorManager.go

@@ -25,6 +25,7 @@ var AvailableCollectors = map[string]MetricCollector{
 	"topprocs":         new(TopProcsCollector),
 	"nvidia":           new(NvidiaCollector),
 	"customcmd":        new(CustomCmdCollector),
+	"iostat":           new(IOstatCollector),
 	"diskstat":         new(DiskstatCollector),
 	"tempstat":         new(TempCollector),
 	"ipmistat":         new(IpmiCollector),

collectors/diskstatMetric.go

@@ -1,18 +1,21 @@
 package collectors
 
 import (
-	"io/ioutil"
+	"bufio"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
-	//	"log"
 	"encoding/json"
-	"errors"
+	"fmt"
-	"strconv"
+	"os"
 	"strings"
+	"syscall"
 	"time"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
 )
 
-const DISKSTATFILE = `/proc/diskstats`
+//	"log"
-const DISKSTAT_SYSFSPATH = `/sys/block`
+
+const MOUNTFILE = `/proc/self/mounts`
 
 type DiskstatCollectorConfig struct {
 	ExcludeMetrics []string `json:"exclude_metrics,omitempty"`
@@ -20,93 +23,89 @@ type DiskstatCollectorConfig struct {
 
 type DiskstatCollector struct {
 	metricCollector
-	matches map[int]string
+	//matches map[string]int
-	config  DiskstatCollectorConfig
+	config IOstatCollectorConfig
+	//devices map[string]IOstatCollectorEntry
 }
 
 func (m *DiskstatCollector) Init(config json.RawMessage) error {
-	var err error
 	m.name = "DiskstatCollector"
 	m.meta = map[string]string{"source": m.name, "group": "Disk"}
 	m.setup()
 	if len(config) > 0 {
-		err = json.Unmarshal(config, &m.config)
+		err := json.Unmarshal(config, &m.config)
 		if err != nil {
 			return err
 		}
 	}
-	// https://www.kernel.org/doc/html/latest/admin-guide/iostats.html
+	file, err := os.Open(string(MOUNTFILE))
-	matches := map[int]string{
+	if err != nil {
-		3:  "reads",
+		cclog.ComponentError(m.name, err.Error())
-		4:  "reads_merged",
-		5:  "read_sectors",
-		6:  "read_ms",
-		7:  "writes",
-		8:  "writes_merged",
-		9:  "writes_sectors",
-		10: "writes_ms",
-		11: "ioops",
-		12: "ioops_ms",
-		13: "ioops_weighted_ms",
-		14: "discards",
-		15: "discards_merged",
-		16: "discards_sectors",
-		17: "discards_ms",
-		18: "flushes",
-		19: "flushes_ms",
-	}
-	m.matches = make(map[int]string)
-	for k, v := range matches {
-		_, skip := stringArrayContains(m.config.ExcludeMetrics, v)
-		if !skip {
-			m.matches[k] = v
-		}
-	}
-	if len(m.matches) == 0 {
-		return errors.New("No metrics to collect")
-	}
-	_, err = ioutil.ReadFile(string(DISKSTATFILE))
-	if err == nil {
-		m.init = true
-	}
 		return err
+	}
+	defer file.Close()
+	m.init = true
+	return nil
 }
 
 func (m *DiskstatCollector) Read(interval time.Duration, output chan lp.CCMetric) {
-	var lines []string
 	if !m.init {
 		return
 	}
 
-	buffer, err := ioutil.ReadFile(string(DISKSTATFILE))
+	file, err := os.Open(string(MOUNTFILE))
 	if err != nil {
+		cclog.ComponentError(m.name, err.Error())
 		return
 	}
-	lines = strings.Split(string(buffer), "\n")
+	defer file.Close()
 
-	for _, line := range lines {
+	part_max_used := uint64(0)
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
 		if len(line) == 0 {
 			continue
 		}
-		f := strings.Fields(line)
+		if !strings.HasPrefix(line, "/dev") {
-		if strings.Contains(f[2], "loop") {
 			continue
 		}
-		tags := map[string]string{
+		linefields := strings.Fields(line)
-			"device": f[2],
+		if strings.Contains(linefields[0], "loop") {
-			"type":   "node",
+			continue
 		}
-		for idx, name := range m.matches {
+		if strings.Contains(linefields[1], "boot") {
-			if idx < len(f) {
+			continue
-				x, err := strconv.ParseInt(f[idx], 0, 64)
+		}
-				if err == nil {
+		path := strings.Replace(linefields[1], `\040`, " ", -1)
-					y, err := lp.New(name, tags, m.meta, map[string]interface{}{"value": int(x)}, time.Now())
+		stat := syscall.Statfs_t{}
+		err := syscall.Statfs(path, &stat)
+		if err != nil {
+			fmt.Println(err.Error())
+			return
+		}
+		tags := map[string]string{"type": "node", "device": linefields[0]}
+		total := (stat.Blocks * uint64(stat.Bsize)) / uint64(1000000000)
+		y, err := lp.New("disk_total", tags, m.meta, map[string]interface{}{"value": total}, time.Now())
 		if err == nil {
+			y.AddMeta("unit", "GBytes")
 			output <- y
 		}
+		free := (stat.Bfree * uint64(stat.Bsize)) / uint64(1000000000)
+		y, err = lp.New("disk_free", tags, m.meta, map[string]interface{}{"value": free}, time.Now())
+		if err == nil {
+			y.AddMeta("unit", "GBytes")
+			output <- y
+		}
+		perc := (100 * (total - free)) / total
+		if perc > part_max_used {
+			part_max_used = perc
 		}
 	}
-		}
+	y, err := lp.New("part_max_used", map[string]string{"type": "node"}, m.meta, map[string]interface{}{"value": part_max_used}, time.Now())
+	if err == nil {
+		y.AddMeta("unit", "percent")
+		output <- y
 	}
 }
 

collectors/diskstatMetric.md

@@ -4,31 +4,18 @@
 ```json
   "diskstat": {
     "exclude_metrics": [
-      "read_ms"
+      "disk_total"
     ],
   }
 ```
 
-The `netstat` collector reads data from `/proc/net/dev` and outputs a handful **node** metrics. If a metric is not required, it can be excluded from forwarding it to the sink.
+The `diskstat` collector reads data from `/proc/self/mounts` and outputs a handful **node** metrics. If a metric is not required, it can be excluded from forwarding it to the sink.
 
-Metrics:
+Metrics per device (with `device` tag):
-* `reads`
+* `disk_total` (unit `GBytes`)
-* `reads_merged`
+* `disk_free` (unit `GBytes`)
-* `read_sectors`
+
-* `read_ms`
+Global metrics:
-* `writes`
+* `part_max_used` (unit `percent`)
-* `writes_merged`
-* `writes_sectors`
-* `writes_ms`
-* `ioops`
-* `ioops_ms`
-* `ioops_weighted_ms`
-* `discards`
-* `discards_merged`
-* `discards_sectors`
-* `discards_ms`
-* `flushes`
-* `flushes_ms`
 
-The device name is added as tag `device`.
 

collectors/iostatMetric.go

@@ -0,0 +1,155 @@
+package collectors
+
+import (
+	"bufio"
+	"os"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+
+	//	"log"
+	"encoding/json"
+	"errors"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const IOSTATFILE = `/proc/diskstats`
+const IOSTAT_SYSFSPATH = `/sys/block`
+
+type IOstatCollectorConfig struct {
+	ExcludeMetrics []string `json:"exclude_metrics,omitempty"`
+}
+
+type IOstatCollectorEntry struct {
+	lastValues map[string]int64
+	tags       map[string]string
+}
+
+type IOstatCollector struct {
+	metricCollector
+	matches map[string]int
+	config  IOstatCollectorConfig
+	devices map[string]IOstatCollectorEntry
+}
+
+func (m *IOstatCollector) Init(config json.RawMessage) error {
+	var err error
+	m.name = "IOstatCollector"
+	m.meta = map[string]string{"source": m.name, "group": "Disk"}
+	m.setup()
+	if len(config) > 0 {
+		err = json.Unmarshal(config, &m.config)
+		if err != nil {
+			return err
+		}
+	}
+	// https://www.kernel.org/doc/html/latest/admin-guide/iostats.html
+	matches := map[string]int{
+		"io_reads":             3,
+		"io_reads_merged":      4,
+		"io_read_sectors":      5,
+		"io_read_ms":           6,
+		"io_writes":            7,
+		"io_writes_merged":     8,
+		"io_writes_sectors":    9,
+		"io_writes_ms":         10,
+		"io_ioops":             11,
+		"io_ioops_ms":          12,
+		"io_ioops_weighted_ms": 13,
+		"io_discards":          14,
+		"io_discards_merged":   15,
+		"io_discards_sectors":  16,
+		"io_discards_ms":       17,
+		"io_flushes":           18,
+		"io_flushes_ms":        19,
+	}
+	m.devices = make(map[string]IOstatCollectorEntry)
+	m.matches = make(map[string]int)
+	for k, v := range matches {
+		if _, skip := stringArrayContains(m.config.ExcludeMetrics, k); !skip {
+			m.matches[k] = v
+		}
+	}
+	if len(m.matches) == 0 {
+		return errors.New("no metrics to collect")
+	}
+	file, err := os.Open(string(IOSTATFILE))
+	if err != nil {
+		cclog.ComponentError(m.name, err.Error())
+		return err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		linefields := strings.Fields(line)
+		device := linefields[2]
+		if strings.Contains(device, "loop") {
+			continue
+		}
+		values := make(map[string]int64)
+		for m := range m.matches {
+			values[m] = 0
+		}
+		m.devices[device] = IOstatCollectorEntry{
+			tags: map[string]string{
+				"device": linefields[2],
+				"type":   "node",
+			},
+			lastValues: values,
+		}
+	}
+	m.init = true
+	return err
+}
+
+func (m *IOstatCollector) Read(interval time.Duration, output chan lp.CCMetric) {
+	if !m.init {
+		return
+	}
+
+	file, err := os.Open(string(IOSTATFILE))
+	if err != nil {
+		cclog.ComponentError(m.name, err.Error())
+		return
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
+		}
+		linefields := strings.Fields(line)
+		device := linefields[2]
+		if strings.Contains(device, "loop") {
+			continue
+		}
+		if _, ok := m.devices[device]; !ok {
+			continue
+		}
+		entry := m.devices[device]
+		for name, idx := range m.matches {
+			if idx < len(linefields) {
+				x, err := strconv.ParseInt(linefields[idx], 0, 64)
+				if err == nil {
+					diff := x - entry.lastValues[name]
+					y, err := lp.New(name, entry.tags, m.meta, map[string]interface{}{"value": int(diff)}, time.Now())
+					if err == nil {
+						output <- y
+					}
+				}
+				entry.lastValues[name] = x
+			}
+		}
+		m.devices[device] = entry
+	}
+}
+
+func (m *IOstatCollector) Close() {
+	m.init = false
+}

collectors/iostatMetric.md

@@ -0,0 +1,34 @@
+
+## `iostat` collector
+
+```json
+  "iostat": {
+    "exclude_metrics": [
+      "read_ms"
+    ],
+  }
+```
+
+The `iostat` collector reads data from `/proc/diskstats` and outputs a handful **node** metrics. If a metric is not required, it can be excluded from forwarding it to the sink.
+
+Metrics:
+* `io_reads`
+* `io_reads_merged`
+* `io_read_sectors`
+* `io_read_ms`
+* `io_writes`
+* `io_writes_merged`
+* `io_writes_sectors`
+* `io_writes_ms`
+* `io_ioops`
+* `io_ioops_ms`
+* `io_ioops_weighted_ms`
+* `io_discards`
+* `io_discards_merged`
+* `io_discards_sectors`
+* `io_discards_ms`
+* `io_flushes`
+* `io_flushes_ms`
+
+The device name is added as tag `device`. For more details, see https://www.kernel.org/doc/html/latest/admin-guide/iostats.html
+