ClusterCockpit/cc-metric-collector
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-metric-collector.git synced 2025-01-13 07:39:05 +01:00
Code Issues Packages Projects Releases Wiki Activity

Formatting

Browse Source
This commit is contained in:
Thomas Roehl 2021-11-25 15:11:39 +01:00
parent a88ea4fd43
commit 9232dd9732
15 changed files with 577 additions and 566 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
collectors/Makefile
View File

@ -1,14 +1,20 @@
# Use central installation
CENTRAL_INSTALL = false
CENTRAL_INSTALL = true
# Path to central installation (if CENTRAL_INSTALL=true)
LIKWID_BASE=/usr/local
LIKWID_BASE=/apps/likwid/5.2.0
# LIKWID version (should be same major version as central installation, 5.1.x)
LIKWID_VERSION = 5.1.0
# Target user for LIKWID's accessdaemon (if CENTRAL_INSTALL=false)
DAEMON_USER=root
# Target group for LIKWID's accessdaemon (if CENTRAL_INSTALL=false)
DAEMON_GROUP=root
LIKWID_VERSION = 5.2.0
#######################################################################
# if CENTRAL_INSTALL=false
#######################################################################
# Target user for LIKWID's accessdaemon
DAEMON_USER=unrz139
# Target group for LIKWID's accessdaemon
DAEMON_GROUP=unrz
# Folder for the group files
GROUP_FOLDER=/usr/local/share/likwid/perfgroups/
#################################################
@ -21,7 +27,7 @@ ifneq ($(strip $(CENTRAL_INSTALL)),true)
LIKWID_BASE := $(shell pwd)/$(INSTALL_FOLDER)
DAEMON_BASE := $(LIKWID_BASE)
GROUPS_BASE := $(LIKWID_BASE)/groups
all: $(INSTALL_FOLDER)/liblikwid.a $(INSTALL_FOLDER)/likwid-accessD cleanup prepare_collector
all: $(INSTALL_FOLDER)/liblikwid.a cleanup prepare_collector
else
DAEMON_BASE= $(LIKWID_BASE)/sbin
GROUPS_BASE := $(LIKWID_BASE)/share/likwid/perfgroups
@ -45,6 +51,7 @@ $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION): $(BUILD_FOLDER)/likwid-$(LIKWID_VERSIO
$(INSTALL_FOLDER)/liblikwid.a: $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION) $(INSTALL_FOLDER)
sed -i -e s+"PREFIX ?= .*"+"PREFIX = $(LIKWID_BASE)"+g \
-e s+"SHARED_LIBRARY = .*"+"SHARED_LIBRARY = false"+g \
-e s+"ACCESSMODE = .*"+"ACCESSMODE = accessdaemon"+g \
-e s+"INSTALLED_ACCESSDAEMON = .*"+"INSTALLED_ACCESSDAEMON = $(DAEMON_BASE)/likwid-accessD"+g \
-e s+"LIKWIDGROUPPATH = .*"+"LIKWIDGROUPPATH = $(GROUPS_BASE)"+g \
$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/config.mk
@ -64,3 +71,8 @@ prepare_collector: likwidMetric.go
cleanup:
rm -rf $(BUILD_FOLDER)
clean: cleanup
rm -rf likwid
.PHONY: clean

5
collectors/cpustatMetric.go
View File

@ -1,13 +1,13 @@
package collectors
import (
"encoding/json"
"fmt"
lp "github.com/influxdata/line-protocol"
"io/ioutil"
"strconv"
"strings"
"time"
"encoding/json"
)
const CPUSTATFILE = `/proc/stat`
@ -34,7 +34,6 @@ func (m *CpustatCollector) Init(config []byte) error {
return nil
}
func ParseStatLine(line string, cpu int, exclude []string, out *[]lp.MutableMetric) {
ls := strings.Fields(line)
matches := []string{"", "cpu_user", "cpu_nice", "cpu_system", "cpu_idle", "cpu_iowait", "cpu_irq", "cpu_softirq", "cpu_steal", "cpu_guest", "cpu_guest_nice"}
@ -62,7 +61,7 @@ func ParseStatLine(line string, cpu int, exclude []string, out *[]lp.MutableMetr
}
func (m *CpustatCollector) Read(interval time.Duration, out *[]lp.MutableMetric) {
if (!m.init) {
if !m.init {
return
}
buffer, err := ioutil.ReadFile(string(CPUSTATFILE))

4
collectors/customCmdMetric.go
View File

@ -1,14 +1,14 @@
package collectors
import (
"encoding/json"
"errors"
lp "github.com/influxdata/line-protocol"
"io/ioutil"
"log"
"os/exec"
"time"
"encoding/json"
"strings"
"time"
)
const CUSTOMCMDPATH = `/home/unrz139/Work/cc-metric-collector/collectors/custom`

10
collectors/diskstatMetric.go
View File

@ -3,12 +3,12 @@ package collectors
import (
lp "github.com/influxdata/line-protocol"
"io/ioutil"
// "log"
// "log"
"encoding/json"
"errors"
"strconv"
"strings"
"time"
"encoding/json"
"errors"
)
const DISKSTATFILE = `/proc/diskstats`
@ -24,7 +24,6 @@ type DiskstatCollector struct {
config DiskstatCollectorConfig
}
func (m *DiskstatCollector) Init(config []byte) error {
var err error
m.name = "DiskstatCollector"
@ -58,7 +57,7 @@ func (m *DiskstatCollector) Init(config []byte) error {
m.matches = make(map[int]string)
for k, v := range matches {
_, skip := stringArrayContains(m.config.ExcludeMetrics, v)
if (!skip) {
if !skip {
m.matches[k] = v
}
}
@ -72,7 +71,6 @@ func (m *DiskstatCollector) Init(config []byte) error {
return err
}
func (m *DiskstatCollector) Read(interval time.Duration, out *[]lp.MutableMetric) {
var lines []string
if !m.init {

85
collectors/infinibandMetric.go
View File

@ -6,13 +6,13 @@ import (
"io/ioutil"
"log"
"os/exec"
// "os"
// "os"
"encoding/json"
"errors"
"path/filepath"
"strconv"
"strings"
"time"
"encoding/json"
"errors"
)
const BASEPATH = `/sys/class/infiniband/`
@ -46,7 +46,7 @@ func (m *InfinibandCollector) Init(config []byte) error {
m.lids = make(map[string]map[string]string)
p := fmt.Sprintf("%s/*/ports/*/lid", string(BASEPATH))
files, err := filepath.Glob(p)
for _, f := range(files) {
for _, f := range files {
lid, err := ioutil.ReadFile(f)
if err == nil {
plist := strings.Split(strings.Replace(f, string(BASEPATH), "", -1), "/")
@ -69,7 +69,7 @@ func (m *InfinibandCollector) Init(config []byte) error {
command := exec.Command(PERFQUERY, args)
command.Wait()
_, err := command.Output()
if (err == nil) {
if err == nil {
m.use_perfquery = true
}
break
@ -155,7 +155,7 @@ func (m *InfinibandCollector) Read(interval time.Duration, out *[]lp.MutableMetr
if m.init {
for dev, ports := range m.lids {
for port, lid := range ports {
tags := map[string]string{"type" : "node", "device" : dev, "port" : port}
tags := map[string]string{"type": "node", "device": dev, "port": port}
if m.use_perfquery {
DoPerfQuery(dev, lid, port, tags, out)
} else {
@ -165,48 +165,47 @@ func (m *InfinibandCollector) Read(interval time.Duration, out *[]lp.MutableMetr
}
}
// buffer, err := ioutil.ReadFile(string(LIDFILE))
// buffer, err := ioutil.ReadFile(string(LIDFILE))
// if err != nil {
// log.Print(err)
// return
// }
// if err != nil {
// log.Print(err)
// return
// }
// args := fmt.Sprintf("-r %s 1 0xf000", string(buffer))
// args := fmt.Sprintf("-r %s 1 0xf000", string(buffer))
// command := exec.Command(PERFQUERY, args)
// command.Wait()
// stdout, err := command.Output()
// if err != nil {
// log.Print(err)
// return
// }
// command := exec.Command(PERFQUERY, args)
// command.Wait()
// stdout, err := command.Output()
// if err != nil {
// log.Print(err)
// return
// }
// ll := strings.Split(string(stdout), "\n")
// ll := strings.Split(string(stdout), "\n")
// for _, line := range ll {
// if strings.HasPrefix(line, "PortRcvData") || strings.HasPrefix(line, "RcvData") {
// lv := strings.Fields(line)
// v, err := strconv.ParseFloat(lv[1], 64)
// if err == nil {
// y, err := lp.New("ib_recv", m.tags, map[string]interface{}{"value": float64(v)}, time.Now())
// if err == nil {
// *out = append(*out, y)
// }
// }
// }
// if strings.HasPrefix(line, "PortXmitData") || strings.HasPrefix(line, "XmtData") {
// lv := strings.Fields(line)
// v, err := strconv.ParseFloat(lv[1], 64)
// if err == nil {
// y, err := lp.New("ib_xmit", m.tags, map[string]interface{}{"value": float64(v)}, time.Now())
// if err == nil {
// *out = append(*out, y)
// }
// }
// }
// }
// for _, line := range ll {
// if strings.HasPrefix(line, "PortRcvData") || strings.HasPrefix(line, "RcvData") {
// lv := strings.Fields(line)
// v, err := strconv.ParseFloat(lv[1], 64)
// if err == nil {
// y, err := lp.New("ib_recv", m.tags, map[string]interface{}{"value": float64(v)}, time.Now())
// if err == nil {
// *out = append(*out, y)
// }
// }
// }
// if strings.HasPrefix(line, "PortXmitData") || strings.HasPrefix(line, "XmtData") {
// lv := strings.Fields(line)
// v, err := strconv.ParseFloat(lv[1], 64)
// if err == nil {
// y, err := lp.New("ib_xmit", m.tags, map[string]interface{}{"value": float64(v)}, time.Now())
// if err == nil {
// *out = append(*out, y)
// }
// }
// }
// }
}
func (m *InfinibandCollector) Close() {

26
collectors/likwidMetric.go
View File

@ -9,19 +9,19 @@ package collectors
import "C"
import (
"encoding/json"
"errors"
"fmt"
lp "github.com/influxdata/line-protocol"
"log"
"strings"
"time"
"os"
"unsafe"
"math"
"encoding/json"
"gopkg.in/Knetic/govaluate.v2"
"io/ioutil"
"log"
"math"
"os"
"strconv"
"strings"
"time"
"unsafe"
)
type LikwidCollectorMetricConfig struct {
@ -40,6 +40,7 @@ type LikwidCollectorConfig struct {
Eventsets []LikwidCollectorEventsetConfig `json:"eventsets"`
Metrics []LikwidCollectorMetricConfig `json:"globalmetrics"`
ExcludeMetrics []string `json:"exclude_metrics"`
ForceOverwrite bool `json:"force_overwrite"`
}
type LikwidCollector struct {
@ -106,7 +107,7 @@ func getBaseFreq() float64 {
data := strings.Replace(string(buffer), "\n", "", -1)
x, err := strconv.ParseInt(data, 0, 64)
if err == nil {
freq = float64(x)*1E3
freq = float64(x) * 1e3
}
}
}
@ -155,7 +156,9 @@ func (m *LikwidCollector) Init(config []byte) error {
if ret != 0 {
return errors.New("Failed to initialize LIKWID topology")
}
if m.config.ForceOverwrite {
os.Setenv("LIKWID_FORCE", "1")
}
ret = C.perfmon_init(C.int(len(m.cpulist)), &m.cpulist[0])
if ret != 0 {
C.topology_finalize()
@ -223,14 +226,14 @@ func (m *LikwidCollector) Read(interval time.Duration, out *[]lp.MutableMetric)
m.results[i][tid][gctr] = float64(res)
}
m.results[i][tid]["time"] = float64(interval)
m.results[i][tid]["inverseClock"] = float64(1.0/m.basefreq)
m.results[i][tid]["inverseClock"] = float64(1.0 / m.basefreq)
for _, metric := range evset.Metrics {
expression, err := govaluate.NewEvaluableExpression(metric.Calc)
if err != nil {
log.Print(err.Error())
continue
}
result, err := expression.Evaluate(m.results[i][tid]);
result, err := expression.Evaluate(m.results[i][tid])
if err != nil {
log.Print(err.Error())
continue
@ -254,7 +257,7 @@ func (m *LikwidCollector) Read(interval time.Duration, out *[]lp.MutableMetric)
params[mname] = mres
}
}
result, err := expression.Evaluate(params);
result, err := expression.Evaluate(params)
if err != nil {
log.Print(err.Error())
continue
@ -319,7 +322,6 @@ func (m *LikwidCollector) Read(interval time.Duration, out *[]lp.MutableMetric)
}
}
func (m *LikwidCollector) Close() {
if m.init {
m.init = false

2
collectors/loadavgMetric.go
View File

@ -1,12 +1,12 @@
package collectors
import (
"encoding/json"
lp "github.com/influxdata/line-protocol"
"io/ioutil"
"strconv"
"strings"
"time"
"encoding/json"
)
const LOADAVGFILE = `/proc/loadavg`

4
collectors/lustreMetric.go
View File

@ -1,14 +1,14 @@
package collectors
import (
"encoding/json"
"errors"
lp "github.com/influxdata/line-protocol"
"io/ioutil"
"log"
"strconv"
"strings"
"time"
"encoding/json"
"errors"
)
const LUSTREFILE = `/proc/fs/lustre/llite/lnec-XXXXXX/stats`

4
collectors/memstatMetric.go
View File

@ -1,6 +1,7 @@
package collectors
import (
"encoding/json"
"errors"
"fmt"
lp "github.com/influxdata/line-protocol"
@ -9,7 +10,6 @@ import (
"strconv"
"strings"
"time"
"encoding/json"
)
const MEMSTATFILE = `/proc/meminfo`
@ -49,7 +49,7 @@ func (m *MemstatCollector) Init(config []byte) error {
"SwapFree": "swap_free"}
for k, v := range matches {
_, skip := stringArrayContains(m.config.ExcludeMetrics, k)
if (!skip) {
if !skip {
m.matches[k] = v
}
}

3
collectors/metricCollector.go
View File

@ -1,13 +1,13 @@
package collectors
import (
"errors"
lp "github.com/influxdata/line-protocol"
"io/ioutil"
"log"
"strconv"
"strings"
"time"
"errors"
)
type MetricGetter interface {
@ -151,4 +151,3 @@ func RemoveFromStringList(s []string, r string) ([]string, error) {
}
return s, errors.New("No such string in list")
}

4
collectors/netstatMetric.go
View File

@ -1,13 +1,13 @@
package collectors
import (
"encoding/json"
lp "github.com/influxdata/line-protocol"
"io/ioutil"
"log"
"strconv"
"strings"
"time"
"encoding/json"
)
const NETSTATFILE = `/proc/net/dev`
@ -66,7 +66,7 @@ func (m *NetstatCollector) Read(interval time.Duration, out *[]lp.MutableMetric)
if cont {
continue
}
tags := map[string]string{"device" : dev, "type": "node"}
tags := map[string]string{"device": dev, "type": "node"}
for i, name := range m.matches {
v, err := strconv.ParseInt(f[i], 10, 0)
if err == nil {

4
collectors/nvidiaMetric.go
View File

@ -1,13 +1,13 @@
package collectors
import (
"encoding/json"
"errors"
"fmt"
"github.com/NVIDIA/go-nvml/pkg/nvml"
lp "github.com/influxdata/line-protocol"
"log"
"time"
"encoding/json"
)
type NvidiaCollectorConfig struct {
@ -58,7 +58,7 @@ func (m *NvidiaCollector) Init(config []byte) error {
}
func (m *NvidiaCollector) Read(interval time.Duration, out *[]lp.MutableMetric) {
if (!m.init) {
if !m.init {
return
}
for i := 0; i < m.num_gpus; i++ {

4
collectors/topprocsMetric.go
View File

@ -1,14 +1,14 @@
package collectors
import (
"encoding/json"
"errors"
"fmt"
lp "github.com/influxdata/line-protocol"
"log"
"os/exec"
"strings"
"time"
"encoding/json"
"errors"
)
const MAX_NUM_PROCS = 10

2
config.json
View File

@ -42,6 +42,8 @@
]
},
"likwid": {
"accessmode" : "accessdaemon",
"daemon_path" : "/apps/likwid/5.2.0/sbin",
"eventsets": [
{
"events": {

2
metric-collector.go
View File

@ -240,7 +240,7 @@ func main() {
}
err = col.Init([]byte(conf))
if err != nil {
log.Print("SKIP ", col.Name(), " (", err.Error(),")")
log.Print("SKIP ", col.Name(), " (", err.Error(), ")")
} else {
log.Print("Start ", col.Name())
tmp = append(tmp, c)