Add include configuration to diskstat collector

collectors/diskstatMetric.go

@@ -22,14 +22,23 @@ import (
 const MOUNTFILE = `/proc/self/mounts`
 
 type DiskstatCollectorConfig struct {
-	ExcludeMetrics []string `json:"exclude_metrics,omitempty"`
-	ExcludeMounts  []string `json:"exclude_mounts,omitempty"`
+	ExcludeMetrics     []string `json:"exclude_metrics,omitempty"`
+	ExcludeDevices     []string `json:"exclude_devices,omitempty"`
+	ExcludeMountpoints []string `json:"exclude_mountpoints,omitempty"`
+	IncludeDevices     []string `json:"include_devices,omitempty"`
+	IncludeMountpoints []string `json:"include_mountpoints,omitempty"`
+	UseMountpoint      bool     `json:"mountpoint_as_stype,omitempty"`
+	UseIncludeConfig   bool     `json:"use_include_config,omitempty"`
 }
 
 type DiskstatCollector struct {
 	metricCollector
-	config         DiskstatCollectorConfig
-	allowedMetrics map[string]bool
+	config             DiskstatCollectorConfig
+	allowedMetrics     map[string]bool
+	includeDevices     map[string]bool
+	includeMountpoints map[string]bool
+	excludeDevices     map[string]bool
+	excludeMountpoints map[string]bool
 }
 
 func (m *DiskstatCollector) Init(config json.RawMessage) error {
@@ -37,6 +46,7 @@ func (m *DiskstatCollector) Init(config json.RawMessage) error {
 	m.parallel = true
 	m.meta = map[string]string{"source": m.name, "group": "Disk"}
 	m.setup()
+	m.config.UseIncludeConfig = false
 	if len(config) > 0 {
 		if err := json.Unmarshal(config, &m.config); err != nil {
 			return err
@@ -52,12 +62,50 @@ func (m *DiskstatCollector) Init(config json.RawMessage) error {
 			m.allowedMetrics[excl] = false
 		}
 	}
+
 	file, err := os.Open(MOUNTFILE)
 	if err != nil {
 		cclog.ComponentError(m.name, err.Error())
 		return err
 	}
 	defer file.Close()
+	availDevices := make(map[string]struct{})
+	availMpoints := make(map[string]struct{})
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
+		}
+		linefields := strings.Fields(line)
+		availDevices[linefields[0]] = struct{}{}
+		availMpoints[linefields[1]] = struct{}{}
+	}
+	m.includeDevices = make(map[string]bool)
+	for _, incl := range m.config.IncludeDevices {
+		if _, ok := availDevices[incl]; ok {
+			m.includeDevices[incl] = true
+		} else {
+			cclog.ComponentWarn(m.name, "Included Mount device ", incl, " does not exist")
+		}
+	}
+	m.includeMountpoints = make(map[string]bool)
+	for _, incl := range m.config.IncludeMountpoints {
+		if _, ok := availMpoints[incl]; ok {
+			m.includeMountpoints[incl] = true
+		} else {
+			cclog.ComponentWarn(m.name, "Included Mount point ", incl, " does not exist")
+		}
+	}
+	m.excludeMountpoints = make(map[string]bool)
+	for _, excl := range m.config.ExcludeMountpoints {
+		m.excludeMountpoints[excl] = true
+	}
+	m.excludeDevices = make(map[string]bool)
+	for _, excl := range m.config.ExcludeDevices {
+		m.excludeDevices[excl] = true
+	}
+
 	m.init = true
 	return nil
 }
@@ -75,6 +123,8 @@ func (m *DiskstatCollector) Read(interval time.Duration, output chan lp.CCMessag
 	defer file.Close()
 
 	part_max_used := uint64(0)
+	part_max_used_device := ""
+	part_max_used_mountpoint := ""
 	scanner := bufio.NewScanner(file)
 mountLoop:
 	for scanner.Scan() {
@@ -82,9 +132,9 @@ mountLoop:
 		if len(line) == 0 {
 			continue
 		}
-		if !strings.HasPrefix(line, "/dev") {
-			continue
-		}
+		// if !strings.HasPrefix(line, "/dev") {
+		// 	continue
+		// }
 		linefields := strings.Fields(line)
 		if strings.Contains(linefields[0], "loop") {
 			continue
@@ -95,8 +145,16 @@ mountLoop:
 
 		mountPath := strings.Replace(linefields[1], `\040`, " ", -1)
 
-		for _, excl := range m.config.ExcludeMounts {
-			if strings.Contains(mountPath, excl) {
+		if m.config.UseIncludeConfig {
+			_, ok1 := m.includeDevices[linefields[0]]
+			_, ok2 := m.includeMountpoints[linefields[1]]
+			if !(ok1 || ok2) {
+				continue mountLoop
+			}
+		} else {
+			_, ok1 := m.excludeDevices[linefields[0]]
+			_, ok2 := m.excludeMountpoints[linefields[1]]
+			if ok1 || ok2 {
 				continue mountLoop
 			}
 		}
@@ -109,7 +167,10 @@ mountLoop:
 		if stat.Blocks == 0 || stat.Bsize == 0 {
 			continue
 		}
-		tags := map[string]string{"type": "node", "device": linefields[0]}
+		tags := map[string]string{"type": "node", "stype": "filesystem", "stype-id": linefields[0]}
+		if m.config.UseMountpoint {
+			tags["stype-id"] = linefields[1]
+		}
 		total := (stat.Blocks * uint64(stat.Bsize)) / uint64(1000000000)
 		if m.allowedMetrics["disk_total"] {
 			y, err := lp.NewMessage("disk_total", tags, m.meta, map[string]interface{}{"value": total}, time.Now())
@@ -130,11 +191,17 @@ mountLoop:
 			perc := (100 * (total - free)) / total
 			if perc > part_max_used {
 				part_max_used = perc
+				part_max_used_mountpoint = linefields[1]
+				part_max_used_device = linefields[0]
 			}
 		}
 	}
-	if m.allowedMetrics["part_max_used"] {
-		y, err := lp.NewMessage("part_max_used", map[string]string{"type": "node"}, m.meta, map[string]interface{}{"value": int(part_max_used)}, time.Now())
+	if m.allowedMetrics["part_max_used"] && len(part_max_used_mountpoint) > 0 {
+		tags := map[string]string{"type": "node", "stype": "filesystem", "stype-id": part_max_used_device}
+		if m.config.UseMountpoint {
+			tags["stype-id"] = part_max_used_mountpoint
+		}
+		y, err := lp.NewMessage("part_max_used", tags, m.meta, map[string]interface{}{"value": int(part_max_used)}, time.Now())
 		if err == nil {
 			y.AddMeta("unit", "percent")
 			output <- y

collectors/diskstatMetric.md

@@ -16,19 +16,38 @@ hugo_path: docs/reference/cc-metric-collector/collectors/diskstat.md
     "exclude_metrics": [
       "disk_total"
     ],
-    "exclude_mounts": [
+    "exclude_devices": [
       "slurm-tmpfs"
+    ],
+    "exclude_mountpoints": [
+      "/tmp"
+    ],
+    "mountpoint_as_stype": true,
+    "use_include_config": false,
+    "include_devices": [
+      "/dev/sda3"
+    ],
+    "include_mountpoints" : [
+      "/home"
     ]
   }
 ```
 
-The `diskstat` collector reads data from `/proc/self/mounts` and outputs a handful **node** metrics. If a metric is not required, it can be excluded from forwarding it to the sink. Additionally, any mount point containing one of the strings specified in `exclude_mounts` will be skipped during metric collection.
+The `diskstat` collector reads data from `/proc/self/mounts` and outputs a handful **node** metrics with `stype=filesystem,stype-id=<mountdevice>`. If a metric is not required, it can be excluded from forwarding it to the sink.
 
-Metrics per device (with `device` tag):
+For sending the `mountpoint` instead of the `mountdevice` in the `stype-id`, use `mountpoint_as_stype`.
+
+There are two ways to specify for which devices or mountpoints the collector generates metrics. It's "either ...or".
+  - Excluding devices and mount points using `exclude_devices` and `exclude_mountpoints`. All devices (*) will be read that are not explicitly excluded
+  - Include devices and mount points by setting `use_include_config:true` and using `include_devices` and `include_mountpoints`.
+
+(*) File systems where the mount device (first column in `/proc/self/mounts`) contains `loop` are always excluded. Filesystems where the mount point (second column in `/proc/self/mounts`) contains `boot` are also always excluded.
+
+Metrics per filesystem (with `stype=filesystem` tag and `stype-id` based on the configuration):
 * `disk_total` (unit `GBytes`)
 * `disk_free` (unit `GBytes`)
 
-Global metrics:
+Global metrics (with `stype=filesystem` tag and `stype-id` pointing to the max. used filesystem device or mount point based on the configuration):
 * `part_max_used` (unit `percent`)