From b77d9a6cf07439330953cd104eb54cb041a90f2c Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 19 Dec 2025 17:32:54 +0100 Subject: [PATCH] Add include configuration to diskstat collector --- collectors/diskstatMetric.go | 91 +++++++++++++++++++++++++++++++----- collectors/diskstatMetric.md | 27 +++++++++-- 2 files changed, 102 insertions(+), 16 deletions(-) diff --git a/collectors/diskstatMetric.go b/collectors/diskstatMetric.go index 4a68b75..3167595 100644 --- a/collectors/diskstatMetric.go +++ b/collectors/diskstatMetric.go @@ -22,14 +22,23 @@ import ( const MOUNTFILE = `/proc/self/mounts` type DiskstatCollectorConfig struct { - ExcludeMetrics []string `json:"exclude_metrics,omitempty"` - ExcludeMounts []string `json:"exclude_mounts,omitempty"` + ExcludeMetrics []string `json:"exclude_metrics,omitempty"` + ExcludeDevices []string `json:"exclude_devices,omitempty"` + ExcludeMountpoints []string `json:"exclude_mountpoints,omitempty"` + IncludeDevices []string `json:"include_devices,omitempty"` + IncludeMountpoints []string `json:"include_mountpoints,omitempty"` + UseMountpoint bool `json:"mountpoint_as_stype,omitempty"` + UseIncludeConfig bool `json:"use_include_config,omitempty"` } type DiskstatCollector struct { metricCollector - config DiskstatCollectorConfig - allowedMetrics map[string]bool + config DiskstatCollectorConfig + allowedMetrics map[string]bool + includeDevices map[string]bool + includeMountpoints map[string]bool + excludeDevices map[string]bool + excludeMountpoints map[string]bool } func (m *DiskstatCollector) Init(config json.RawMessage) error { @@ -37,6 +46,7 @@ func (m *DiskstatCollector) Init(config json.RawMessage) error { m.parallel = true m.meta = map[string]string{"source": m.name, "group": "Disk"} m.setup() + m.config.UseIncludeConfig = false if len(config) > 0 { if err := json.Unmarshal(config, &m.config); err != nil { return err @@ -52,12 +62,50 @@ func (m *DiskstatCollector) Init(config json.RawMessage) error { m.allowedMetrics[excl] = false } } + file, err := os.Open(MOUNTFILE) if err != nil { cclog.ComponentError(m.name, err.Error()) return err } defer file.Close() + availDevices := make(map[string]struct{}) + availMpoints := make(map[string]struct{}) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + linefields := strings.Fields(line) + availDevices[linefields[0]] = struct{}{} + availMpoints[linefields[1]] = struct{}{} + } + m.includeDevices = make(map[string]bool) + for _, incl := range m.config.IncludeDevices { + if _, ok := availDevices[incl]; ok { + m.includeDevices[incl] = true + } else { + cclog.ComponentWarn(m.name, "Included Mount device ", incl, " does not exist") + } + } + m.includeMountpoints = make(map[string]bool) + for _, incl := range m.config.IncludeMountpoints { + if _, ok := availMpoints[incl]; ok { + m.includeMountpoints[incl] = true + } else { + cclog.ComponentWarn(m.name, "Included Mount point ", incl, " does not exist") + } + } + m.excludeMountpoints = make(map[string]bool) + for _, excl := range m.config.ExcludeMountpoints { + m.excludeMountpoints[excl] = true + } + m.excludeDevices = make(map[string]bool) + for _, excl := range m.config.ExcludeDevices { + m.excludeDevices[excl] = true + } + m.init = true return nil } @@ -75,6 +123,8 @@ func (m *DiskstatCollector) Read(interval time.Duration, output chan lp.CCMessag defer file.Close() part_max_used := uint64(0) + part_max_used_device := "" + part_max_used_mountpoint := "" scanner := bufio.NewScanner(file) mountLoop: for scanner.Scan() { @@ -82,9 +132,9 @@ mountLoop: if len(line) == 0 { continue } - if !strings.HasPrefix(line, "/dev") { - continue - } + // if !strings.HasPrefix(line, "/dev") { + // continue + // } linefields := strings.Fields(line) if strings.Contains(linefields[0], "loop") { continue @@ -95,8 +145,16 @@ mountLoop: mountPath := strings.Replace(linefields[1], `\040`, " ", -1) - for _, excl := range m.config.ExcludeMounts { - if strings.Contains(mountPath, excl) { + if m.config.UseIncludeConfig { + _, ok1 := m.includeDevices[linefields[0]] + _, ok2 := m.includeMountpoints[linefields[1]] + if !(ok1 || ok2) { + continue mountLoop + } + } else { + _, ok1 := m.excludeDevices[linefields[0]] + _, ok2 := m.excludeMountpoints[linefields[1]] + if ok1 || ok2 { continue mountLoop } } @@ -109,7 +167,10 @@ mountLoop: if stat.Blocks == 0 || stat.Bsize == 0 { continue } - tags := map[string]string{"type": "node", "device": linefields[0]} + tags := map[string]string{"type": "node", "stype": "filesystem", "stype-id": linefields[0]} + if m.config.UseMountpoint { + tags["stype-id"] = linefields[1] + } total := (stat.Blocks * uint64(stat.Bsize)) / uint64(1000000000) if m.allowedMetrics["disk_total"] { y, err := lp.NewMessage("disk_total", tags, m.meta, map[string]interface{}{"value": total}, time.Now()) @@ -130,11 +191,17 @@ mountLoop: perc := (100 * (total - free)) / total if perc > part_max_used { part_max_used = perc + part_max_used_mountpoint = linefields[1] + part_max_used_device = linefields[0] } } } - if m.allowedMetrics["part_max_used"] { - y, err := lp.NewMessage("part_max_used", map[string]string{"type": "node"}, m.meta, map[string]interface{}{"value": int(part_max_used)}, time.Now()) + if m.allowedMetrics["part_max_used"] && len(part_max_used_mountpoint) > 0 { + tags := map[string]string{"type": "node", "stype": "filesystem", "stype-id": part_max_used_device} + if m.config.UseMountpoint { + tags["stype-id"] = part_max_used_mountpoint + } + y, err := lp.NewMessage("part_max_used", tags, m.meta, map[string]interface{}{"value": int(part_max_used)}, time.Now()) if err == nil { y.AddMeta("unit", "percent") output <- y diff --git a/collectors/diskstatMetric.md b/collectors/diskstatMetric.md index 2108a21..32f090d 100644 --- a/collectors/diskstatMetric.md +++ b/collectors/diskstatMetric.md @@ -16,19 +16,38 @@ hugo_path: docs/reference/cc-metric-collector/collectors/diskstat.md "exclude_metrics": [ "disk_total" ], - "exclude_mounts": [ + "exclude_devices": [ "slurm-tmpfs" + ], + "exclude_mountpoints": [ + "/tmp" + ], + "mountpoint_as_stype": true, + "use_include_config": false, + "include_devices": [ + "/dev/sda3" + ], + "include_mountpoints" : [ + "/home" ] } ``` -The `diskstat` collector reads data from `/proc/self/mounts` and outputs a handful **node** metrics. If a metric is not required, it can be excluded from forwarding it to the sink. Additionally, any mount point containing one of the strings specified in `exclude_mounts` will be skipped during metric collection. +The `diskstat` collector reads data from `/proc/self/mounts` and outputs a handful **node** metrics with `stype=filesystem,stype-id=`. If a metric is not required, it can be excluded from forwarding it to the sink. -Metrics per device (with `device` tag): +For sending the `mountpoint` instead of the `mountdevice` in the `stype-id`, use `mountpoint_as_stype`. + +There are two ways to specify for which devices or mountpoints the collector generates metrics. It's "either ...or". + - Excluding devices and mount points using `exclude_devices` and `exclude_mountpoints`. All devices (*) will be read that are not explicitly excluded + - Include devices and mount points by setting `use_include_config:true` and using `include_devices` and `include_mountpoints`. + +(*) File systems where the mount device (first column in `/proc/self/mounts`) contains `loop` are always excluded. Filesystems where the mount point (second column in `/proc/self/mounts`) contains `boot` are also always excluded. + +Metrics per filesystem (with `stype=filesystem` tag and `stype-id` based on the configuration): * `disk_total` (unit `GBytes`) * `disk_free` (unit `GBytes`) -Global metrics: +Global metrics (with `stype=filesystem` tag and `stype-id` pointing to the max. used filesystem device or mount point based on the configuration): * `part_max_used` (unit `percent`)