Feedback implemented

collectors/beegfsmetaMetric.go

@@ -14,9 +14,12 @@ import (
 	"strings"
 	"time"
 
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
 )
 
+const BEEGFS_CMD = "beegfs-ctl"
+
 // Struct for the collector-specific JSON config
 type BeegfsMetaCollectorConfig struct {
 	Beegfs            string   `json:"beegfs_path"`
@@ -63,7 +66,7 @@ func (m *BeegfsMetaCollector) Init(config json.RawMessage) error {
 			return err
 		}
 	}
-	println(m.config.Beegfs)
+
 	//create map with possible variables
 	m.matches = make(map[string]string)
 	for _, value := range nodeMdstat_array {
@@ -71,7 +74,7 @@ func (m *BeegfsMetaCollector) Init(config json.RawMessage) error {
 		if skip {
 			m.matches["other"] = "0"
 		} else {
-			m.matches[value] = "0"
+			m.matches["beegfs_cmeta_"+value] = "0"
 		}
 	}
 
@@ -113,7 +116,7 @@ func (m *BeegfsMetaCollector) Read(interval time.Duration, output chan lp.CCMetr
 	//get mounpoint
 	buffer, _ := ioutil.ReadFile(string("/proc/mounts"))
 	mounts := strings.Split(string(buffer), "\n")
-	var mountpoint string
+	var mountpoints []string
 	for _, line := range mounts {
 		if len(line) == 0 {
 			continue
@@ -121,82 +124,103 @@ func (m *BeegfsMetaCollector) Read(interval time.Duration, output chan lp.CCMetr
 		f := strings.Fields(line)
 		if strings.Contains(f[0], "beegfs_ondemand") {
 			// Skip excluded filesystems
-			if _, skip := m.skipFS[mountpoint]; skip {
+			if _, skip := m.skipFS[f[1]]; skip {
 				continue
 			}
-			mountpoint = f[1]
+			mountpoints = append(mountpoints, f[1])
 		}
 	}
-	m.tags["filesystem"] = mountpoint
 
-	// bwwgfs-ctl:
+	if len(mountpoints) == 0 {
-	// --clientstats: Show client IO statistics.
+		cclog.ComponentError(
-	// --nodetype=meta: The node type to query (meta, storage).
+			m.name,
-	// --interval:
+			"Read(): Failed to find BeeGFS on Demand FS.")
-	// --mount=/mnt/beeond/: Which mount point
-	//cmd := exec.Command(m.config.Beegfs, "/root/mc/test.txt")
-	mountoption := "--mount=" + mountpoint
-	cmd := exec.Command(m.config.Beegfs, "--clientstats",
-		"--nodetype=meta", mountoption, "--allstats")
-	cmd.Stdin = strings.NewReader("\n")
-	cmdStdout := new(bytes.Buffer)
-	cmdStderr := new(bytes.Buffer)
-	cmd.Stdout = cmdStdout
-	cmd.Stderr = cmdStderr
-	err := cmd.Run()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error())
-		fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode())
-		data, _ := ioutil.ReadAll(cmdStderr)
-		fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stderr: \"%s\"\n", string(data))
-		data, _ = ioutil.ReadAll(cmdStdout)
-		fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stdout: \"%s\"\n", string(data))
-		return
 	}
-	// Read I/O statistics
-	scanner := bufio.NewScanner(cmdStdout)
 
-	sumLine := regexp.MustCompile(`^Sum:\s+\d+\s+\[[a-zA-Z]+\]+`)
+	for _, mountpoint := range mountpoints {
-	//Line := regexp.MustCompile(`^(.*)\s+(\d)+\s+\[([a-zA-Z]+)\]+`)
+		m.tags["filesystem"] = mountpoint
-	statsLine := regexp.MustCompile(`^(.*?)\s+?(\d.*?)$`)
-	singleSpacePattern := regexp.MustCompile(`\s+`)
-	removePattern := regexp.MustCompile(`[\[|\]]`)
 
-	for scanner.Scan() {
+		// bwwgfs-ctl:
-		readLine := scanner.Text()
+		// --clientstats: Show client IO statistics.
-		//fmt.Println(readLine)
+		// --nodetype=meta: The node type to query (meta, storage).
-		// Jump few lines, we only want the I/O stats from nodes
+		// --interval:
-		if !sumLine.MatchString(readLine) {
+		// --mount=/mnt/beeond/: Which mount point
-			continue
+		//cmd := exec.Command(m.config.Beegfs, "/root/mc/test.txt")
+		mountoption := "--mount=" + mountpoint
+		cmd := exec.Command(m.config.Beegfs, "--clientstats",
+			"--nodetype=meta", mountoption, "--allstats")
+		cmd.Stdin = strings.NewReader("\n")
+		cmdStdout := new(bytes.Buffer)
+		cmdStderr := new(bytes.Buffer)
+		cmd.Stdout = cmdStdout
+		cmd.Stderr = cmdStderr
+		err := cmd.Run()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error())
+			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode())
+			data, _ := ioutil.ReadAll(cmdStderr)
+			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stderr: \"%s\"\n", string(data))
+			data, _ = ioutil.ReadAll(cmdStdout)
+			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stdout: \"%s\"\n", string(data))
+			return
 		}
+		// Read I/O statistics
+		scanner := bufio.NewScanner(cmdStdout)
 
-		match := statsLine.FindStringSubmatch(readLine)
+		sumLine := regexp.MustCompile(`^Sum:\s+\d+\s+\[[a-zA-Z]+\]+`)
-		// nodeName = "Sum:" or would be nodes
+		//Line := regexp.MustCompile(`^(.*)\s+(\d)+\s+\[([a-zA-Z]+)\]+`)
-		// nodeName := match[1]
+		statsLine := regexp.MustCompile(`^(.*?)\s+?(\d.*?)$`)
-		//Remove multiple whitespaces
+		singleSpacePattern := regexp.MustCompile(`\s+`)
-		dummy := removePattern.ReplaceAllString(match[2], " ")
+		removePattern := regexp.MustCompile(`[\[|\]]`)
-		metaStats := strings.TrimSpace(singleSpacePattern.ReplaceAllString(dummy, " "))
-		split := strings.Split(metaStats, " ")
 
-		// fill map with values
+		for scanner.Scan() {
-		// split[i+1] = mdname
+			readLine := scanner.Text()
-		// split[i] = amount of md operations
+			//fmt.Println(readLine)
-		for i := 0; i <= len(split)-1; i += 2 {
+			// Jump few lines, we only want the I/O stats from nodes
-			if _, ok := m.matches[split[i+1]]; ok {
+			if !sumLine.MatchString(readLine) {
-				m.matches[split[i+1]] = split[i]
+				continue
-			} else {
-				f1, _ := strconv.ParseFloat(m.matches["other"], 32)
-				f2, _ := strconv.ParseFloat(split[i], 32)
-				//mdStat["other"] = fmt.Sprintf("%f", f1+f2)
-				m.matches["other"] = fmt.Sprintf("%f", f1+f2)
 			}
-		}
 
-		for key, data := range m.matches {
+			match := statsLine.FindStringSubmatch(readLine)
-			value, _ := strconv.ParseFloat(data, 32)
+			// nodeName = "Sum:" or would be nodes
-			y, err := lp.New(key, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now())
+			// nodeName := match[1]
-			if err == nil {
+			//Remove multiple whitespaces
-				output <- y
+			dummy := removePattern.ReplaceAllString(match[2], " ")
+			metaStats := strings.TrimSpace(singleSpacePattern.ReplaceAllString(dummy, " "))
+			split := strings.Split(metaStats, " ")
+
+			// fill map with values
+			// split[i+1] = mdname
+			// split[i] = amount of md operations
+			for i := 0; i <= len(split)-1; i += 2 {
+				if _, ok := m.matches[split[i+1]]; ok {
+					m.matches["beegfs_cmeta_"+split[i+1]] = split[i]
+				} else {
+					f1, err := strconv.ParseFloat(m.matches["other"], 32)
+					if err != nil {
+						cclog.ComponentError(
+							m.name,
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
+						continue
+					}
+					f2, err := strconv.ParseFloat(split[i], 32)
+					if err != nil {
+						cclog.ComponentError(
+							m.name,
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
+						continue
+					}
+					//mdStat["other"] = fmt.Sprintf("%f", f1+f2)
+					m.matches["beegfs_cstorage_other"] = fmt.Sprintf("%f", f1+f2)
+				}
+			}
+
+			for key, data := range m.matches {
+				value, _ := strconv.ParseFloat(data, 32)
+				y, err := lp.New(key, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now())
+				if err == nil {
+					output <- y
+				}
 			}
 		}
 	}

collectors/beegfsmetaMetric.md

@@ -25,6 +25,10 @@ in the configuration.
 
 When using the `exclude_metrics` option, the excluded metrics are summed as `other`.
 
+Important: The metrics listed below, are similar to the naming of BeeGFS. The Collector prefixes these with `beegfs_cstorage`(beegfs client storage).
+
+For example beegfs metric `open`-> `beegfs_cstorage_open`
+
 Available Metrics:
 
 * sum

collectors/beegfsstorageMetric.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
 )
 
@@ -66,7 +67,7 @@ func (m *BeegfsStorageCollector) Init(config json.RawMessage) error {
 		if skip {
 			m.matches["other"] = "0"
 		} else {
-			m.matches[value] = "0"
+			m.matches["beegfs_cstorage_"+value] = "0"
 		}
 	}
 
@@ -108,7 +109,7 @@ func (m *BeegfsStorageCollector) Read(interval time.Duration, output chan lp.CCM
 	//get mounpoint
 	buffer, _ := ioutil.ReadFile(string("/proc/mounts"))
 	mounts := strings.Split(string(buffer), "\n")
-	var mountpoint string
+	var mountpoints []string
 	for _, line := range mounts {
 		if len(line) == 0 {
 			continue
@@ -116,81 +117,102 @@ func (m *BeegfsStorageCollector) Read(interval time.Duration, output chan lp.CCM
 		f := strings.Fields(line)
 		if strings.Contains(f[0], "beegfs_ondemand") {
 			// Skip excluded filesystems
-			if _, skip := m.skipFS[mountpoint]; skip {
+			if _, skip := m.skipFS[f[1]]; skip {
 				continue
 			}
-			mountpoint = f[1]
+			mountpoints = append(mountpoints, f[1])
 		}
 	}
-	m.tags["filesystem"] = mountpoint
+	if len(mountpoints) == 0 {
-
+		cclog.ComponentError(
-	// bwwgfs-ctl:
+			m.name,
-	// --clientstats: Show client IO statistics.
+			"Read(): Failed to find BeeGFS on Demand FS.")
-	// --nodetype=meta: The node type to query (meta, storage).
-	// --interval:
-	// --mount=/mnt/beeond/: Which mount point
-	//cmd := exec.Command(m.config.Beegfs, "/root/mc/test.txt")
-	mountoption := "--mount=" + mountpoint
-	cmd := exec.Command(m.config.Beegfs, "--clientstats",
-		"--nodetype=storage", mountoption, "--allstats")
-	cmd.Stdin = strings.NewReader("\n")
-	cmdStdout := new(bytes.Buffer)
-	cmdStderr := new(bytes.Buffer)
-	cmd.Stdout = cmdStdout
-	cmd.Stderr = cmdStderr
-	err := cmd.Run()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error())
-		fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode())
-		data, _ := ioutil.ReadAll(cmdStderr)
-		fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stderr: \"%s\"\n", string(data))
-		data, _ = ioutil.ReadAll(cmdStdout)
-		fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stdout: \"%s\"\n", string(data))
-		return
 	}
-	// Read I/O statistics
+	// collects stats for each BeeGFS on Demand FS
-	scanner := bufio.NewScanner(cmdStdout)
+	for _, mountpoint := range mountpoints {
+		m.tags["filesystem"] = mountpoint
 
-	sumLine := regexp.MustCompile(`^Sum:\s+\d+\s+\[[a-zA-Z]+\]+`)
+		// bwwgfs-ctl:
-	//Line := regexp.MustCompile(`^(.*)\s+(\d)+\s+\[([a-zA-Z]+)\]+`)
+		// --clientstats: Show client IO statistics.
-	statsLine := regexp.MustCompile(`^(.*?)\s+?(\d.*?)$`)
+		// --nodetype=meta: The node type to query (meta, storage).
-	singleSpacePattern := regexp.MustCompile(`\s+`)
+		// --interval:
-	removePattern := regexp.MustCompile(`[\[|\]]`)
+		// --mount=/mnt/beeond/: Which mount point
-
+		//cmd := exec.Command(m.config.Beegfs, "/root/mc/test.txt")
-	for scanner.Scan() {
+		mountoption := "--mount=" + mountpoint
-		readLine := scanner.Text()
+		cmd := exec.Command(m.config.Beegfs, "--clientstats",
-		//fmt.Println(readLine)
+			"--nodetype=storage", mountoption, "--allstats")
-		// Jump few lines, we only want the I/O stats from nodes
+		cmd.Stdin = strings.NewReader("\n")
-		if !sumLine.MatchString(readLine) {
+		cmdStdout := new(bytes.Buffer)
-			continue
+		cmdStderr := new(bytes.Buffer)
+		cmd.Stdout = cmdStdout
+		cmd.Stderr = cmdStderr
+		err := cmd.Run()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error())
+			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode())
+			data, _ := ioutil.ReadAll(cmdStderr)
+			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stderr: \"%s\"\n", string(data))
+			data, _ = ioutil.ReadAll(cmdStdout)
+			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stdout: \"%s\"\n", string(data))
+			return
 		}
+		// Read I/O statistics
+		scanner := bufio.NewScanner(cmdStdout)
 
-		match := statsLine.FindStringSubmatch(readLine)
+		sumLine := regexp.MustCompile(`^Sum:\s+\d+\s+\[[a-zA-Z]+\]+`)
-		// nodeName = "Sum:" or would be nodes
+		//Line := regexp.MustCompile(`^(.*)\s+(\d)+\s+\[([a-zA-Z]+)\]+`)
-		// nodeName := match[1]
+		statsLine := regexp.MustCompile(`^(.*?)\s+?(\d.*?)$`)
-		//Remove multiple whitespaces
+		singleSpacePattern := regexp.MustCompile(`\s+`)
-		dummy := removePattern.ReplaceAllString(match[2], " ")
+		removePattern := regexp.MustCompile(`[\[|\]]`)
-		metaStats := strings.TrimSpace(singleSpacePattern.ReplaceAllString(dummy, " "))
-		split := strings.Split(metaStats, " ")
 
-		// fill map with values
+		for scanner.Scan() {
-		// split[i+1] = mdname
+			readLine := scanner.Text()
-		// split[i] = amount of operations
+			//fmt.Println(readLine)
-		for i := 0; i <= len(split)-1; i += 2 {
+			// Jump few lines, we only want the I/O stats from nodes
-			if _, ok := m.matches[split[i+1]]; ok {
+			if !sumLine.MatchString(readLine) {
-				m.matches[split[i+1]] = split[i]
+				continue
-			} else {
-				f1, _ := strconv.ParseFloat(m.matches["other"], 32)
-				f2, _ := strconv.ParseFloat(split[i], 32)
-				m.matches["other"] = fmt.Sprintf("%f", f1+f2)
 			}
-		}
 
-		for key, data := range m.matches {
+			match := statsLine.FindStringSubmatch(readLine)
-			value, _ := strconv.ParseFloat(data, 32)
+			// nodeName = "Sum:" or would be nodes
-			y, err := lp.New(key, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now())
+			// nodeName := match[1]
-			if err == nil {
+			//Remove multiple whitespaces
-				output <- y
+			dummy := removePattern.ReplaceAllString(match[2], " ")
+			metaStats := strings.TrimSpace(singleSpacePattern.ReplaceAllString(dummy, " "))
+			split := strings.Split(metaStats, " ")
+
+			// fill map with values
+			// split[i+1] = mdname
+			// split[i] = amount of operations
+			for i := 0; i <= len(split)-1; i += 2 {
+				if _, ok := m.matches[split[i+1]]; ok {
+					m.matches["beegfs_cstorage_"+split[i+1]] = split[i]
+					//m.matches[split[i+1]] = split[i]
+				} else {
+					f1, err := strconv.ParseFloat(m.matches["other"], 32)
+					if err != nil {
+						cclog.ComponentError(
+							m.name,
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
+						continue
+					}
+					f2, err := strconv.ParseFloat(split[i], 32)
+					if err != nil {
+						cclog.ComponentError(
+							m.name,
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
+						continue
+					}
+					m.matches["beegfs_cstorage_other"] = fmt.Sprintf("%f", f1+f2)
+				}
+			}
+
+			for key, data := range m.matches {
+				value, _ := strconv.ParseFloat(data, 32)
+				y, err := lp.New(key, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now())
+				if err == nil {
+					output <- y
+				}
 			}
 		}
 	}

collectors/beegfsstorageMetric.md

@@ -24,6 +24,11 @@ in the configuration.
 
 When using the `exclude_metrics` option, the excluded metrics are summed as `other`.
 
+Important: The metrics listed below, are similar to the naming of BeeGFS. The Collector prefixes these with `beegfs_cstorage_`(beegfs client meta).
+For example beegfs metric `open`-> `beegfs_cstorage_`
+
+Note: BeeGFS FS offers many Metadata Information. Probably it makes sense to exlcude most of them. Nevertheless, these excluded metrics will be summed as `beegfs_cstorage_other`. 
+
 Available Metrics:
 
 * "sum"