From db02c89683b41b74bf7811ec0dad14e1a9eeda44 Mon Sep 17 00:00:00 2001
From: Thomas Roehl <Thomas.Roehl@googlemail.com>
Date: Thu, 3 Feb 2022 22:05:16 +0100
Subject: [PATCH 1/3] Update LustreCollector to use lctl. Sysfs version is
commented out
---
collectors/lustreMetric.go | 91 +++++++++++++++++++++++++++++---------
1 file changed, 70 insertions(+), 21 deletions(-)
diff --git a/collectors/lustreMetric.go b/collectors/lustreMetric.go
index 3e248fa..99b371c 100644
--- a/collectors/lustreMetric.go
+++ b/collectors/lustreMetric.go
@@ -3,18 +3,21 @@ package collectors
import (
"encoding/json"
"errors"
- "io/ioutil"
- "log"
+ "fmt"
+ "os/exec"
"strconv"
"strings"
"time"
+
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
)
-const LUSTREFILE = `/proc/fs/lustre/llite/lnec-XXXXXX/stats`
+const LUSTRE_SYSFS = `/sys/fs/lustre`
+const LCTL_CMD = `lctl`
+const LCTL_OPTION = `get_param`
type LustreCollectorConfig struct {
- Procfiles []string `json:"procfiles"`
+ LCtlCommand string `json:"lctl_command"`
ExcludeMetrics []string `json:"exclude_metrics"`
}
@@ -24,6 +27,58 @@ type LustreCollector struct {
matches map[string]map[string]int
devices []string
config LustreCollectorConfig
+ lctl string
+}
+
+func (m *LustreCollector) getDevices() []string {
+ devices := make([]string, 0)
+
+ // //Version reading devices from sysfs
+ // globPattern := filepath.Join(LUSTRE_SYSFS, "llite/*/stats")
+ // files, err := filepath.Glob(globPattern)
+ // if err != nil {
+ // return devices
+ // }
+ // for _, f := range files {
+ // pathlist := strings.Split(f, "/")
+ // devices = append(devices, pathlist[4])
+ // }
+
+ command := exec.Command(m.lctl, LCTL_OPTION, "llite.*.stats")
+ command.Wait()
+ stdout, err := command.Output()
+ if err != nil {
+ return devices
+ }
+ for _, line := range strings.Split(string(stdout), "\n") {
+ if strings.HasPrefix(line, "llite") {
+ linefields := strings.Split(line, ".")
+ if len(linefields) > 2 {
+ devices = append(devices, linefields[1])
+ }
+ }
+ }
+ return devices
+}
+
+// //Version reading the stats data of a device from sysfs
+// func (m *LustreCollector) getDeviceDataSysfs(device string) []string {
+// llitedir := filepath.Join(LUSTRE_SYSFS, "llite")
+// devdir := filepath.Join(llitedir, device)
+// statsfile := filepath.Join(devdir, "stats")
+// buffer, err := ioutil.ReadFile(statsfile)
+// if err != nil {
+// return make([]string, 0)
+// }
+// return strings.Split(string(buffer), "\n")
+// }
+
+func (m *LustreCollector) getDeviceDataCommand(device string) []string {
+ statsfile := fmt.Sprintf("llite.%s.stats", device)
+ command := exec.Command(m.lctl, LCTL_OPTION, statsfile)
+ command.Wait()
+ stdout, _ := command.Output()
+ return strings.Split(string(stdout), "\n")
}
func (m *LustreCollector) Init(config json.RawMessage) error {
@@ -46,19 +101,18 @@ func (m *LustreCollector) Init(config json.RawMessage) error {
"getattr": {"getattr": 1},
"statfs": {"statfs": 1},
"inode_permission": {"inode_permission": 1}}
- m.devices = make([]string, 0)
- for _, p := range m.config.Procfiles {
- _, err := ioutil.ReadFile(p)
- if err == nil {
- m.devices = append(m.devices, p)
- } else {
- log.Print(err.Error())
- continue
+ p, err := exec.LookPath(m.config.LCtlCommand)
+ if err != nil {
+ p, err = exec.LookPath(LCTL_CMD)
+ if err != nil {
+ return err
}
}
+ m.lctl = p
+ m.devices = m.getDevices()
if len(m.devices) == 0 {
- return errors.New("No metrics to collect")
+ return errors.New("no metrics to collect")
}
m.init = true
return nil
@@ -68,15 +122,10 @@ func (m *LustreCollector) Read(interval time.Duration, output chan lp.CCMetric)
if !m.init {
return
}
- for _, p := range m.devices {
- buffer, err := ioutil.ReadFile(p)
+ for _, device := range m.devices {
+ stats := m.getDeviceDataCommand(device)
- if err != nil {
- log.Print(err)
- return
- }
-
- for _, line := range strings.Split(string(buffer), "\n") {
+ for _, line := range stats {
lf := strings.Fields(line)
if len(lf) > 1 {
for match, fields := range m.matches {
From 02cd21abe24110867a5a1a1b75bceb288caf8af6 Mon Sep 17 00:00:00 2001
From: Thomas Roehl <Thomas.Roehl@googlemail.com>
Date: Fri, 4 Feb 2022 12:39:25 +0100
Subject: [PATCH 2/3] HTTPS for HttpSink
---
sinks/httpSink.go | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/sinks/httpSink.go b/sinks/httpSink.go
index 25b0082..a703f82 100644
--- a/sinks/httpSink.go
+++ b/sinks/httpSink.go
@@ -26,7 +26,11 @@ func (s *HttpSink) Init(config sinkConfig) error {
}
s.client = &http.Client{}
- s.url = fmt.Sprintf("http://%s:%s/%s", config.Host, config.Port, config.Database)
+ proto := "http"
+ if config.SSL {
+ proto = "https"
+ }
+ s.url = fmt.Sprintf("%s://%s:%s/%s", proto, config.Host, config.Port, config.Database)
s.port = config.Port
s.jwt = config.Password
s.buffer = &bytes.Buffer{}
From 66b9a25a88ba759f53d5d83a9bcb73119766076c Mon Sep 17 00:00:00 2001
From: Thomas Roehl <Thomas.Roehl@googlemail.com>
Date: Fri, 4 Feb 2022 12:39:59 +0100
Subject: [PATCH 3/3] Prefix metrics from NetstatCollector with 'net'
---
collectors/netstatMetric.go | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/collectors/netstatMetric.go b/collectors/netstatMetric.go
index 86437ea..0ec94a4 100644
--- a/collectors/netstatMetric.go
+++ b/collectors/netstatMetric.go
@@ -7,6 +7,7 @@ import (
"strconv"
"strings"
"time"
+
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
)
@@ -27,10 +28,10 @@ func (m *NetstatCollector) Init(config json.RawMessage) error {
m.setup()
m.meta = map[string]string{"source": m.name, "group": "Memory"}
m.matches = map[int]string{
- 1: "bytes_in",
- 9: "bytes_out",
- 2: "pkts_in",
- 10: "pkts_out",
+ 1: "net_bytes_in",
+ 9: "net_bytes_out",
+ 2: "net_pkts_in",
+ 10: "net_pkts_out",
}
if len(config) > 0 {
err := json.Unmarshal(config, &m.config)