From e08b1bd2bdf19791ca649c6f31d45375b2416d20 Mon Sep 17 00:00:00 2001 From: Mehmet Soysal Date: Fri, 4 Mar 2022 11:33:09 +0100 Subject: [PATCH] Feedback implemented --- collectors/beegfsmetaMetric.go | 156 +++++++++++++++++------------- collectors/beegfsmetaMetric.md | 4 + collectors/beegfsstorageMetric.go | 154 ++++++++++++++++------------- collectors/beegfsstorageMetric.md | 5 + 4 files changed, 187 insertions(+), 132 deletions(-) diff --git a/collectors/beegfsmetaMetric.go b/collectors/beegfsmetaMetric.go index 7a96e7f..2b86ee9 100644 --- a/collectors/beegfsmetaMetric.go +++ b/collectors/beegfsmetaMetric.go @@ -14,9 +14,12 @@ import ( "strings" "time" + cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" ) +const BEEGFS_CMD = "beegfs-ctl" + // Struct for the collector-specific JSON config type BeegfsMetaCollectorConfig struct { Beegfs string `json:"beegfs_path"` @@ -63,7 +66,7 @@ func (m *BeegfsMetaCollector) Init(config json.RawMessage) error { return err } } - println(m.config.Beegfs) + //create map with possible variables m.matches = make(map[string]string) for _, value := range nodeMdstat_array { @@ -71,7 +74,7 @@ func (m *BeegfsMetaCollector) Init(config json.RawMessage) error { if skip { m.matches["other"] = "0" } else { - m.matches[value] = "0" + m.matches["beegfs_cmeta_"+value] = "0" } } @@ -113,7 +116,7 @@ func (m *BeegfsMetaCollector) Read(interval time.Duration, output chan lp.CCMetr //get mounpoint buffer, _ := ioutil.ReadFile(string("/proc/mounts")) mounts := strings.Split(string(buffer), "\n") - var mountpoint string + var mountpoints []string for _, line := range mounts { if len(line) == 0 { continue @@ -121,82 +124,103 @@ func (m *BeegfsMetaCollector) Read(interval time.Duration, output chan lp.CCMetr f := strings.Fields(line) if strings.Contains(f[0], "beegfs_ondemand") { // Skip excluded filesystems - if _, skip := m.skipFS[mountpoint]; skip { + if _, skip := m.skipFS[f[1]]; skip { continue } - mountpoint = f[1] + mountpoints = append(mountpoints, f[1]) } } - m.tags["filesystem"] = mountpoint - // bwwgfs-ctl: - // --clientstats: Show client IO statistics. - // --nodetype=meta: The node type to query (meta, storage). - // --interval: - // --mount=/mnt/beeond/: Which mount point - //cmd := exec.Command(m.config.Beegfs, "/root/mc/test.txt") - mountoption := "--mount=" + mountpoint - cmd := exec.Command(m.config.Beegfs, "--clientstats", - "--nodetype=meta", mountoption, "--allstats") - cmd.Stdin = strings.NewReader("\n") - cmdStdout := new(bytes.Buffer) - cmdStderr := new(bytes.Buffer) - cmd.Stdout = cmdStdout - cmd.Stderr = cmdStderr - err := cmd.Run() - if err != nil { - fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error()) - fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode()) - data, _ := ioutil.ReadAll(cmdStderr) - fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stderr: \"%s\"\n", string(data)) - data, _ = ioutil.ReadAll(cmdStdout) - fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stdout: \"%s\"\n", string(data)) - return + if len(mountpoints) == 0 { + cclog.ComponentError( + m.name, + "Read(): Failed to find BeeGFS on Demand FS.") } - // Read I/O statistics - scanner := bufio.NewScanner(cmdStdout) - sumLine := regexp.MustCompile(`^Sum:\s+\d+\s+\[[a-zA-Z]+\]+`) - //Line := regexp.MustCompile(`^(.*)\s+(\d)+\s+\[([a-zA-Z]+)\]+`) - statsLine := regexp.MustCompile(`^(.*?)\s+?(\d.*?)$`) - singleSpacePattern := regexp.MustCompile(`\s+`) - removePattern := regexp.MustCompile(`[\[|\]]`) + for _, mountpoint := range mountpoints { + m.tags["filesystem"] = mountpoint - for scanner.Scan() { - readLine := scanner.Text() - //fmt.Println(readLine) - // Jump few lines, we only want the I/O stats from nodes - if !sumLine.MatchString(readLine) { - continue + // bwwgfs-ctl: + // --clientstats: Show client IO statistics. + // --nodetype=meta: The node type to query (meta, storage). + // --interval: + // --mount=/mnt/beeond/: Which mount point + //cmd := exec.Command(m.config.Beegfs, "/root/mc/test.txt") + mountoption := "--mount=" + mountpoint + cmd := exec.Command(m.config.Beegfs, "--clientstats", + "--nodetype=meta", mountoption, "--allstats") + cmd.Stdin = strings.NewReader("\n") + cmdStdout := new(bytes.Buffer) + cmdStderr := new(bytes.Buffer) + cmd.Stdout = cmdStdout + cmd.Stderr = cmdStderr + err := cmd.Run() + if err != nil { + fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error()) + fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode()) + data, _ := ioutil.ReadAll(cmdStderr) + fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stderr: \"%s\"\n", string(data)) + data, _ = ioutil.ReadAll(cmdStdout) + fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stdout: \"%s\"\n", string(data)) + return } + // Read I/O statistics + scanner := bufio.NewScanner(cmdStdout) - match := statsLine.FindStringSubmatch(readLine) - // nodeName = "Sum:" or would be nodes - // nodeName := match[1] - //Remove multiple whitespaces - dummy := removePattern.ReplaceAllString(match[2], " ") - metaStats := strings.TrimSpace(singleSpacePattern.ReplaceAllString(dummy, " ")) - split := strings.Split(metaStats, " ") + sumLine := regexp.MustCompile(`^Sum:\s+\d+\s+\[[a-zA-Z]+\]+`) + //Line := regexp.MustCompile(`^(.*)\s+(\d)+\s+\[([a-zA-Z]+)\]+`) + statsLine := regexp.MustCompile(`^(.*?)\s+?(\d.*?)$`) + singleSpacePattern := regexp.MustCompile(`\s+`) + removePattern := regexp.MustCompile(`[\[|\]]`) - // fill map with values - // split[i+1] = mdname - // split[i] = amount of md operations - for i := 0; i <= len(split)-1; i += 2 { - if _, ok := m.matches[split[i+1]]; ok { - m.matches[split[i+1]] = split[i] - } else { - f1, _ := strconv.ParseFloat(m.matches["other"], 32) - f2, _ := strconv.ParseFloat(split[i], 32) - //mdStat["other"] = fmt.Sprintf("%f", f1+f2) - m.matches["other"] = fmt.Sprintf("%f", f1+f2) + for scanner.Scan() { + readLine := scanner.Text() + //fmt.Println(readLine) + // Jump few lines, we only want the I/O stats from nodes + if !sumLine.MatchString(readLine) { + continue } - } - for key, data := range m.matches { - value, _ := strconv.ParseFloat(data, 32) - y, err := lp.New(key, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now()) - if err == nil { - output <- y + match := statsLine.FindStringSubmatch(readLine) + // nodeName = "Sum:" or would be nodes + // nodeName := match[1] + //Remove multiple whitespaces + dummy := removePattern.ReplaceAllString(match[2], " ") + metaStats := strings.TrimSpace(singleSpacePattern.ReplaceAllString(dummy, " ")) + split := strings.Split(metaStats, " ") + + // fill map with values + // split[i+1] = mdname + // split[i] = amount of md operations + for i := 0; i <= len(split)-1; i += 2 { + if _, ok := m.matches[split[i+1]]; ok { + m.matches["beegfs_cmeta_"+split[i+1]] = split[i] + } else { + f1, err := strconv.ParseFloat(m.matches["other"], 32) + if err != nil { + cclog.ComponentError( + m.name, + fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err)) + continue + } + f2, err := strconv.ParseFloat(split[i], 32) + if err != nil { + cclog.ComponentError( + m.name, + fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err)) + continue + } + //mdStat["other"] = fmt.Sprintf("%f", f1+f2) + m.matches["beegfs_cstorage_other"] = fmt.Sprintf("%f", f1+f2) + } + } + + for key, data := range m.matches { + value, _ := strconv.ParseFloat(data, 32) + y, err := lp.New(key, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now()) + if err == nil { + output <- y + } } } } diff --git a/collectors/beegfsmetaMetric.md b/collectors/beegfsmetaMetric.md index 1157898..fa3bca4 100644 --- a/collectors/beegfsmetaMetric.md +++ b/collectors/beegfsmetaMetric.md @@ -25,6 +25,10 @@ in the configuration. When using the `exclude_metrics` option, the excluded metrics are summed as `other`. +Important: The metrics listed below, are similar to the naming of BeeGFS. The Collector prefixes these with `beegfs_cstorage`(beegfs client storage). + +For example beegfs metric `open`-> `beegfs_cstorage_open` + Available Metrics: * sum diff --git a/collectors/beegfsstorageMetric.go b/collectors/beegfsstorageMetric.go index 0d54159..a0e4ec4 100644 --- a/collectors/beegfsstorageMetric.go +++ b/collectors/beegfsstorageMetric.go @@ -14,6 +14,7 @@ import ( "strings" "time" + cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger" lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric" ) @@ -66,7 +67,7 @@ func (m *BeegfsStorageCollector) Init(config json.RawMessage) error { if skip { m.matches["other"] = "0" } else { - m.matches[value] = "0" + m.matches["beegfs_cstorage_"+value] = "0" } } @@ -108,7 +109,7 @@ func (m *BeegfsStorageCollector) Read(interval time.Duration, output chan lp.CCM //get mounpoint buffer, _ := ioutil.ReadFile(string("/proc/mounts")) mounts := strings.Split(string(buffer), "\n") - var mountpoint string + var mountpoints []string for _, line := range mounts { if len(line) == 0 { continue @@ -116,81 +117,102 @@ func (m *BeegfsStorageCollector) Read(interval time.Duration, output chan lp.CCM f := strings.Fields(line) if strings.Contains(f[0], "beegfs_ondemand") { // Skip excluded filesystems - if _, skip := m.skipFS[mountpoint]; skip { + if _, skip := m.skipFS[f[1]]; skip { continue } - mountpoint = f[1] + mountpoints = append(mountpoints, f[1]) } } - m.tags["filesystem"] = mountpoint - - // bwwgfs-ctl: - // --clientstats: Show client IO statistics. - // --nodetype=meta: The node type to query (meta, storage). - // --interval: - // --mount=/mnt/beeond/: Which mount point - //cmd := exec.Command(m.config.Beegfs, "/root/mc/test.txt") - mountoption := "--mount=" + mountpoint - cmd := exec.Command(m.config.Beegfs, "--clientstats", - "--nodetype=storage", mountoption, "--allstats") - cmd.Stdin = strings.NewReader("\n") - cmdStdout := new(bytes.Buffer) - cmdStderr := new(bytes.Buffer) - cmd.Stdout = cmdStdout - cmd.Stderr = cmdStderr - err := cmd.Run() - if err != nil { - fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error()) - fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode()) - data, _ := ioutil.ReadAll(cmdStderr) - fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stderr: \"%s\"\n", string(data)) - data, _ = ioutil.ReadAll(cmdStdout) - fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stdout: \"%s\"\n", string(data)) - return + if len(mountpoints) == 0 { + cclog.ComponentError( + m.name, + "Read(): Failed to find BeeGFS on Demand FS.") } - // Read I/O statistics - scanner := bufio.NewScanner(cmdStdout) + // collects stats for each BeeGFS on Demand FS + for _, mountpoint := range mountpoints { + m.tags["filesystem"] = mountpoint - sumLine := regexp.MustCompile(`^Sum:\s+\d+\s+\[[a-zA-Z]+\]+`) - //Line := regexp.MustCompile(`^(.*)\s+(\d)+\s+\[([a-zA-Z]+)\]+`) - statsLine := regexp.MustCompile(`^(.*?)\s+?(\d.*?)$`) - singleSpacePattern := regexp.MustCompile(`\s+`) - removePattern := regexp.MustCompile(`[\[|\]]`) - - for scanner.Scan() { - readLine := scanner.Text() - //fmt.Println(readLine) - // Jump few lines, we only want the I/O stats from nodes - if !sumLine.MatchString(readLine) { - continue + // bwwgfs-ctl: + // --clientstats: Show client IO statistics. + // --nodetype=meta: The node type to query (meta, storage). + // --interval: + // --mount=/mnt/beeond/: Which mount point + //cmd := exec.Command(m.config.Beegfs, "/root/mc/test.txt") + mountoption := "--mount=" + mountpoint + cmd := exec.Command(m.config.Beegfs, "--clientstats", + "--nodetype=storage", mountoption, "--allstats") + cmd.Stdin = strings.NewReader("\n") + cmdStdout := new(bytes.Buffer) + cmdStderr := new(bytes.Buffer) + cmd.Stdout = cmdStdout + cmd.Stderr = cmdStderr + err := cmd.Run() + if err != nil { + fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error()) + fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode()) + data, _ := ioutil.ReadAll(cmdStderr) + fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stderr: \"%s\"\n", string(data)) + data, _ = ioutil.ReadAll(cmdStdout) + fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stdout: \"%s\"\n", string(data)) + return } + // Read I/O statistics + scanner := bufio.NewScanner(cmdStdout) - match := statsLine.FindStringSubmatch(readLine) - // nodeName = "Sum:" or would be nodes - // nodeName := match[1] - //Remove multiple whitespaces - dummy := removePattern.ReplaceAllString(match[2], " ") - metaStats := strings.TrimSpace(singleSpacePattern.ReplaceAllString(dummy, " ")) - split := strings.Split(metaStats, " ") + sumLine := regexp.MustCompile(`^Sum:\s+\d+\s+\[[a-zA-Z]+\]+`) + //Line := regexp.MustCompile(`^(.*)\s+(\d)+\s+\[([a-zA-Z]+)\]+`) + statsLine := regexp.MustCompile(`^(.*?)\s+?(\d.*?)$`) + singleSpacePattern := regexp.MustCompile(`\s+`) + removePattern := regexp.MustCompile(`[\[|\]]`) - // fill map with values - // split[i+1] = mdname - // split[i] = amount of operations - for i := 0; i <= len(split)-1; i += 2 { - if _, ok := m.matches[split[i+1]]; ok { - m.matches[split[i+1]] = split[i] - } else { - f1, _ := strconv.ParseFloat(m.matches["other"], 32) - f2, _ := strconv.ParseFloat(split[i], 32) - m.matches["other"] = fmt.Sprintf("%f", f1+f2) + for scanner.Scan() { + readLine := scanner.Text() + //fmt.Println(readLine) + // Jump few lines, we only want the I/O stats from nodes + if !sumLine.MatchString(readLine) { + continue } - } - for key, data := range m.matches { - value, _ := strconv.ParseFloat(data, 32) - y, err := lp.New(key, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now()) - if err == nil { - output <- y + match := statsLine.FindStringSubmatch(readLine) + // nodeName = "Sum:" or would be nodes + // nodeName := match[1] + //Remove multiple whitespaces + dummy := removePattern.ReplaceAllString(match[2], " ") + metaStats := strings.TrimSpace(singleSpacePattern.ReplaceAllString(dummy, " ")) + split := strings.Split(metaStats, " ") + + // fill map with values + // split[i+1] = mdname + // split[i] = amount of operations + for i := 0; i <= len(split)-1; i += 2 { + if _, ok := m.matches[split[i+1]]; ok { + m.matches["beegfs_cstorage_"+split[i+1]] = split[i] + //m.matches[split[i+1]] = split[i] + } else { + f1, err := strconv.ParseFloat(m.matches["other"], 32) + if err != nil { + cclog.ComponentError( + m.name, + fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err)) + continue + } + f2, err := strconv.ParseFloat(split[i], 32) + if err != nil { + cclog.ComponentError( + m.name, + fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err)) + continue + } + m.matches["beegfs_cstorage_other"] = fmt.Sprintf("%f", f1+f2) + } + } + + for key, data := range m.matches { + value, _ := strconv.ParseFloat(data, 32) + y, err := lp.New(key, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now()) + if err == nil { + output <- y + } } } } diff --git a/collectors/beegfsstorageMetric.md b/collectors/beegfsstorageMetric.md index 322c635..0f379b3 100644 --- a/collectors/beegfsstorageMetric.md +++ b/collectors/beegfsstorageMetric.md @@ -24,6 +24,11 @@ in the configuration. When using the `exclude_metrics` option, the excluded metrics are summed as `other`. +Important: The metrics listed below, are similar to the naming of BeeGFS. The Collector prefixes these with `beegfs_cstorage_`(beegfs client meta). +For example beegfs metric `open`-> `beegfs_cstorage_` + +Note: BeeGFS FS offers many Metadata Information. Probably it makes sense to exlcude most of them. Nevertheless, these excluded metrics will be summed as `beegfs_cstorage_other`. + Available Metrics: * "sum"