From 558bbaba59e4ad8bfacb198f51ce84b53ae138f7 Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Mon, 4 Oct 2021 15:23:43 +0200 Subject: [PATCH 01/11] Change storage format --- collectors/cpustatMetric.go | 51 +++-- collectors/infinibandMetric.go | 19 +- collectors/likwidMetric.go | 71 ++++-- collectors/loadavgMetric.go | 40 +++- collectors/lustreMetric.go | 50 +++-- collectors/memstatMetric.go | 69 ++++-- collectors/metricCollector.go | 56 ++--- collectors/netstatMetric.go | 37 ++-- collectors/nvidiaMetric.go | 135 +++++++++--- collectors/topprocsMetric.go | 18 +- config.json | 18 +- go.mod | 4 + go.sum | 386 +++++++++++++++++++++++++++++++++ metric-collector.go | 106 +++------ receivers/natsReceiver.go | 13 +- sinks/influxSink.go | 14 +- sinks/metricSink.go | 5 +- sinks/natsSink.go | 28 ++- sinks/stdoutSink.go | 37 +++- 19 files changed, 874 insertions(+), 283 deletions(-) diff --git a/collectors/cpustatMetric.go b/collectors/cpustatMetric.go index 6aaaf42..9b428b7 100644 --- a/collectors/cpustatMetric.go +++ b/collectors/cpustatMetric.go @@ -1,6 +1,8 @@ package collectors import ( + "fmt" + lp "github.com/influxdata/line-protocol" "io/ioutil" "strconv" "strings" @@ -16,34 +18,34 @@ type CpustatCollector struct { func (m *CpustatCollector) Init() error { m.name = "CpustatCollector" m.setup() + m.init = true return nil } -func ParseStatLine(line string, out map[string]interface{}) { +func ParseStatLine(line string, cpu int, out *[]lp.MutableMetric) { ls := strings.Fields(line) - user, _ := strconv.ParseInt(ls[1], 0, 64) - out["cpu_user"] = float64(user) - nice, _ := strconv.ParseInt(ls[2], 0, 64) - out["cpu_nice"] = float64(nice) - system, _ := strconv.ParseInt(ls[3], 0, 64) - out["cpu_system"] = float64(system) - idle, _ := strconv.ParseInt(ls[4], 0, 64) - out["cpu_idle"] = float64(idle) - iowait, _ := strconv.ParseInt(ls[5], 0, 64) - out["cpu_iowait"] = float64(iowait) - irq, _ := strconv.ParseInt(ls[6], 0, 64) - out["cpu_irq"] = float64(irq) - softirq, _ := strconv.ParseInt(ls[7], 0, 64) - out["cpu_softirq"] = float64(softirq) - steal, _ := strconv.ParseInt(ls[8], 0, 64) - out["cpu_steal"] = float64(steal) - guest, _ := strconv.ParseInt(ls[9], 0, 64) - out["cpu_guest"] = float64(guest) - guest_nice, _ := strconv.ParseInt(ls[10], 0, 64) - out["cpu_guest_nice"] = float64(guest_nice) + matches := []string{"", "cpu_user", "cpu_nice", "cpu_system", "cpu_idle", "cpu_iowait", "cpu_irq", "cpu_softirq", "cpu_steal", "cpu_guest", "cpu_guest_nice"} + + var tags map[string]string + if cpu < 0 { + tags = map[string]string{"type": "node"} + } else { + tags = map[string]string{"type": "cpu", "type-id": fmt.Sprintf("%d", cpu)} + } + for i, m := range matches { + if len(m) > 0 { + x, err := strconv.ParseInt(ls[i], 0, 64) + if err == nil { + y, err := lp.New(m, tags, map[string]interface{}{"value": int(x)}, time.Now()) + if err == nil { + *out = append(*out, y) + } + } + } + } } -func (m *CpustatCollector) Read(interval time.Duration) { +func (m *CpustatCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { buffer, err := ioutil.ReadFile(string(CPUSTATFILE)) if err != nil { @@ -57,15 +59,16 @@ func (m *CpustatCollector) Read(interval time.Duration) { } ls := strings.Fields(line) if strings.Compare(ls[0], "cpu") == 0 { - ParseStatLine(line, m.node) + ParseStatLine(line, -1, out) } else if strings.HasPrefix(ls[0], "cpu") { cpustr := strings.TrimLeft(ls[0], "cpu") cpu, _ := strconv.Atoi(cpustr) - ParseStatLine(line, m.cpus[cpu]) + ParseStatLine(line, cpu, out) } } } func (m *CpustatCollector) Close() { + m.init = false return } diff --git a/collectors/infinibandMetric.go b/collectors/infinibandMetric.go index 4878fff..099bd5b 100644 --- a/collectors/infinibandMetric.go +++ b/collectors/infinibandMetric.go @@ -2,6 +2,7 @@ package collectors import ( "fmt" + lp "github.com/influxdata/line-protocol" "io/ioutil" "log" "os/exec" @@ -14,16 +15,21 @@ const LIDFILE = `/sys/class/infiniband/mlx4_0/ports/1/lid` type InfinibandCollector struct { MetricCollector + tags map[string]string } func (m *InfinibandCollector) Init() error { m.name = "InfinibandCollector" m.setup() + m.tags = map[string]string{"type": "node"} _, err := ioutil.ReadFile(string(LIDFILE)) + if err == nil { + m.init = true + } return err } -func (m *InfinibandCollector) Read(interval time.Duration) { +func (m *InfinibandCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { buffer, err := ioutil.ReadFile(string(LIDFILE)) if err != nil { @@ -48,19 +54,26 @@ func (m *InfinibandCollector) Read(interval time.Duration) { lv := strings.Fields(line) v, err := strconv.ParseFloat(lv[1], 64) if err == nil { - m.node["ib_recv"] = float64(v) + y, err := lp.New("ib_recv", m.tags, map[string]interface{}{"value": float64(v)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } } if strings.HasPrefix(line, "PortXmitData") || strings.HasPrefix(line, "XmtData") { lv := strings.Fields(line) v, err := strconv.ParseFloat(lv[1], 64) if err == nil { - m.node["ib_xmit"] = float64(v) + y, err := lp.New("ib_xmit", m.tags, map[string]interface{}{"value": float64(v)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } } } } func (m *InfinibandCollector) Close() { + m.init = false return } diff --git a/collectors/likwidMetric.go b/collectors/likwidMetric.go index 5cba77f..4a71c71 100644 --- a/collectors/likwidMetric.go +++ b/collectors/likwidMetric.go @@ -11,6 +11,7 @@ import "C" import ( "errors" "fmt" + lp "github.com/influxdata/line-protocol" "log" "strings" "time" @@ -23,7 +24,6 @@ type LikwidCollector struct { sock2tid map[int]int metrics map[C.int]map[string]int groups map[string]C.int - init bool } type LikwidMetric struct { @@ -132,9 +132,13 @@ func (m *LikwidCollector) Init() error { return nil } -func (m *LikwidCollector) Read(interval time.Duration) { +func (m *LikwidCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { if m.init { var ret C.int + core_fp_any := make(map[int]float64, len(m.cpulist)) + for _, cpu := range m.cpulist { + core_fp_any[int(cpu)] = 0.0 + } for gname, gid := range m.groups { ret = C.perfmon_setupCounters(gid) if ret != 0 { @@ -154,37 +158,68 @@ func (m *LikwidCollector) Read(interval time.Duration) { } for _, lmetric := range likwid_metrics[gname] { + if lmetric.name == "pwr1" || lmetric.name == "pwr2" { + continue + } if lmetric.socket_scope { for sid, tid := range m.sock2tid { res := C.perfmon_getLastMetric(gid, C.int(lmetric.group_idx), C.int(tid)) - m.sockets[int(sid)][lmetric.name] = float64(res) + y, err := lp.New(lmetric.name, + map[string]string{"type": "socket", "type-id": fmt.Sprintf("%d", int(sid))}, + map[string]interface{}{"value": float64(res)}, + time.Now()) + if err == nil { + *out = append(*out, y) + } // log.Print("Metric '", lmetric.name,"' on Socket ",int(sid)," returns ", m.sockets[int(sid)][lmetric.name]) } } else { for tid, cpu := range m.cpulist { res := C.perfmon_getLastMetric(gid, C.int(lmetric.group_idx), C.int(tid)) - m.cpus[int(cpu)][lmetric.name] = float64(res) + y, err := lp.New(lmetric.name, + map[string]string{"type": "cpu", "type-id": fmt.Sprintf("%d", int(cpu))}, + map[string]interface{}{"value": float64(res)}, + time.Now()) + if err == nil { + *out = append(*out, y) + } + if lmetric.name == "flops_dp" { + core_fp_any[int(cpu)] += 2 * float64(res) + } + if lmetric.name == "flops_sp" { + core_fp_any[int(cpu)] += float64(res) + } // log.Print("Metric '", lmetric.name,"' on CPU ",int(cpu)," returns ", m.cpus[int(cpu)][lmetric.name]) } } } - for cpu := range m.cpus { - if flops_dp, found := m.cpus[cpu]["flops_dp"]; found { - if flops_sp, found := m.cpus[cpu]["flops_sp"]; found { - m.cpus[cpu]["flops_any"] = (2 * flops_dp.(float64)) + flops_sp.(float64) + for sid, tid := range m.sock2tid { + sum := 0.0 + valid := false + for _, lmetric := range likwid_metrics[gname] { + if lmetric.name == "pwr1" || lmetric.name == "pwr2" { + res := C.perfmon_getLastMetric(gid, C.int(lmetric.group_idx), C.int(tid)) + sum += float64(res) + valid = true + } + } + if valid { + y, err := lp.New("power", + map[string]string{"type": "socket", "type-id": fmt.Sprintf("%d", int(sid))}, + map[string]interface{}{"value": float64(sum)}, + time.Now()) + if err == nil { + *out = append(*out, y) } } } - for sid := range m.sockets { - if pwr1, found := m.sockets[int(sid)]["pwr1"]; found { - if pwr2, found := m.sockets[int(sid)]["pwr2"]; found { - sum := pwr1.(float64) + pwr2.(float64) - if sum > 0 { - m.sockets[int(sid)]["power"] = sum - } - delete(m.sockets[int(sid)], "pwr2") - } - delete(m.sockets[int(sid)], "pwr1") + for cpu := range m.cpulist { + y, err := lp.New("flops_any", + map[string]string{"type": "cpu", "type-id": fmt.Sprintf("%d", int(cpu))}, + map[string]interface{}{"value": float64(core_fp_any[int(cpu)])}, + time.Now()) + if err == nil { + *out = append(*out, y) } } } diff --git a/collectors/loadavgMetric.go b/collectors/loadavgMetric.go index 291b37d..4be7c87 100644 --- a/collectors/loadavgMetric.go +++ b/collectors/loadavgMetric.go @@ -1,6 +1,7 @@ package collectors import ( + lp "github.com/influxdata/line-protocol" "io/ioutil" "strconv" "strings" @@ -11,15 +12,23 @@ const LOADAVGFILE = `/proc/loadavg` type LoadavgCollector struct { MetricCollector + tags map[string]string + load_matches []string + proc_matches []string } func (m *LoadavgCollector) Init() error { m.name = "LoadavgCollector" m.setup() + m.tags = map[string]string{"type": "node"} + m.load_matches = []string{"load_one", "load_five", "load_fifteen"} + m.proc_matches = []string{"proc_run", "proc_total"} + m.init = true return nil } -func (m *LoadavgCollector) Read(interval time.Duration) { +func (m *LoadavgCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { + buffer, err := ioutil.ReadFile(string(LOADAVGFILE)) if err != nil { @@ -27,19 +36,28 @@ func (m *LoadavgCollector) Read(interval time.Duration) { } ls := strings.Split(string(buffer), ` `) - loadOne, _ := strconv.ParseFloat(ls[0], 64) - m.node["load_one"] = float64(loadOne) - loadFive, _ := strconv.ParseFloat(ls[1], 64) - m.node["load_five"] = float64(loadFive) - loadFifteen, _ := strconv.ParseFloat(ls[2], 64) - m.node["load_fifteen"] = float64(loadFifteen) + for i, name := range m.load_matches { + x, err := strconv.ParseFloat(ls[i], 64) + if err == nil { + y, err := lp.New(name, m.tags, map[string]interface{}{"value": float64(x)}, time.Now()) + if err == nil { + *out = append(*out, y) + } + } + } lv := strings.Split(ls[3], `/`) - proc_run, _ := strconv.ParseFloat(lv[0], 64) - proc_total, _ := strconv.ParseFloat(lv[1], 64) - m.node["proc_total"] = float64(proc_total) - m.node["proc_run"] = float64(proc_run) + for i, name := range m.proc_matches { + x, err := strconv.ParseFloat(lv[i], 64) + if err == nil { + y, err := lp.New(name, m.tags, map[string]interface{}{"value": float64(x)}, time.Now()) + if err == nil { + *out = append(*out, y) + } + } + } } func (m *LoadavgCollector) Close() { + m.init = false return } diff --git a/collectors/lustreMetric.go b/collectors/lustreMetric.go index 26d40fb..a4af598 100644 --- a/collectors/lustreMetric.go +++ b/collectors/lustreMetric.go @@ -1,6 +1,7 @@ package collectors import ( + lp "github.com/influxdata/line-protocol" "io/ioutil" "log" "strconv" @@ -12,16 +13,30 @@ const LUSTREFILE = `/proc/fs/lustre/llite/lnec-XXXXXX/stats` type LustreCollector struct { MetricCollector + tags map[string]string + matches map[string]map[string]int } func (m *LustreCollector) Init() error { m.name = "LustreCollector" m.setup() + m.tags = map[string]string{"type": "node"} + m.matches = map[string]map[string]int{"read_bytes": {"read_bytes": 6, "read_requests": 1}, + "write_bytes": {"write_bytes": 6, "write_requests": 1}, + "open": {"open": 1}, + "close": {"close": 1}, + "setattr": {"setattr": 1}, + "getattr": {"getattr": 1}, + "statfs": {"statfs": 1}, + "inode_permission": {"inode_permission": 1}} _, err := ioutil.ReadFile(string(LUSTREFILE)) + if err == nil { + m.init = true + } return err } -func (m *LustreCollector) Read(interval time.Duration) { +func (m *LustreCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { buffer, err := ioutil.ReadFile(string(LUSTREFILE)) if err != nil { @@ -32,31 +47,24 @@ func (m *LustreCollector) Read(interval time.Duration) { for _, line := range strings.Split(string(buffer), "\n") { lf := strings.Fields(line) if len(lf) > 1 { - switch lf[0] { - case "read_bytes": - m.node["read_bytes"], err = strconv.ParseInt(lf[6], 0, 64) - m.node["read_requests"], err = strconv.ParseInt(lf[1], 0, 64) - case "write_bytes": - m.node["write_bytes"], err = strconv.ParseInt(lf[6], 0, 64) - m.node["write_requests"], err = strconv.ParseInt(lf[1], 0, 64) - case "open": - m.node["open"], err = strconv.ParseInt(lf[1], 0, 64) - case "close": - m.node["close"], err = strconv.ParseInt(lf[1], 0, 64) - case "setattr": - m.node["setattr"], err = strconv.ParseInt(lf[1], 0, 64) - case "getattr": - m.node["getattr"], err = strconv.ParseInt(lf[1], 0, 64) - case "statfs": - m.node["statfs"], err = strconv.ParseInt(lf[1], 0, 64) - case "inode_permission": - m.node["inode_permission"], err = strconv.ParseInt(lf[1], 0, 64) + for match, fields := range m.matches { + if lf[0] == match { + for name, idx := range fields { + x, err := strconv.ParseInt(lf[idx], 0, 64) + if err == nil { + y, err := lp.New(name, m.tags, map[string]interface{}{"value": x}, time.Now()) + if err == nil { + *out = append(*out, y) + } + } + } + } } - } } } func (m *LustreCollector) Close() { + m.init = false return } diff --git a/collectors/memstatMetric.go b/collectors/memstatMetric.go index 0355167..5462de4 100644 --- a/collectors/memstatMetric.go +++ b/collectors/memstatMetric.go @@ -2,6 +2,8 @@ package collectors import ( "errors" + "fmt" + lp "github.com/influxdata/line-protocol" "io/ioutil" "log" "strconv" @@ -13,15 +15,33 @@ const MEMSTATFILE = `/proc/meminfo` type MemstatCollector struct { MetricCollector + stats map[string]int64 + tags map[string]string + matches map[string]string } func (m *MemstatCollector) Init() error { m.name = "MemstatCollector" + m.stats = make(map[string]int64) + m.tags = map[string]string{"type": "node"} + m.matches = map[string]string{`MemTotal`: "mem_total", + "SwapTotal": "swap_total", + "SReclaimable": "mem_sreclaimable", + "Slab": "mem_slab", + "MemFree": "mem_free", + "Buffers": "mem_buffers", + "Cached": "mem_cached", + "MemAvailable": "mem_available", + "SwapFree": "swap_free"} m.setup() + _, err := ioutil.ReadFile(string(MEMSTATFILE)) + if err == nil { + m.init = true + } return nil } -func (m *MemstatCollector) Read(interval time.Duration) { +func (m *MemstatCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { buffer, err := ioutil.ReadFile(string(MEMSTATFILE)) if err != nil { @@ -30,40 +50,53 @@ func (m *MemstatCollector) Read(interval time.Duration) { } ll := strings.Split(string(buffer), "\n") - memstats := make(map[string]int64) for _, line := range ll { ls := strings.Split(line, `:`) if len(ls) > 1 { lv := strings.Fields(ls[1]) - memstats[ls[0]], err = strconv.ParseInt(lv[0], 0, 64) + m.stats[ls[0]], err = strconv.ParseInt(lv[0], 0, 64) } } - if _, exists := memstats[`MemTotal`]; !exists { + if _, exists := m.stats[`MemTotal`]; !exists { err = errors.New("Parse error") log.Print(err) return } - m.node["mem_total"] = float64(memstats[`MemTotal`]) * 1.0e-3 - m.node["swap_total"] = float64(memstats[`SwapTotal`]) * 1.0e-3 - m.node["mem_sreclaimable"] = float64(memstats[`SReclaimable`]) * 1.0e-3 - m.node["mem_slab"] = float64(memstats[`Slab`]) * 1.0e-3 - m.node["mem_free"] = float64(memstats[`MemFree`]) * 1.0e-3 - m.node["mem_buffers"] = float64(memstats[`Buffers`]) * 1.0e-3 - m.node["mem_cached"] = float64(memstats[`Cached`]) * 1.0e-3 - m.node["mem_available"] = float64(memstats[`MemAvailable`]) * 1.0e-3 - m.node["swap_free"] = float64(memstats[`SwapFree`]) * 1.0e-3 + for match, name := range m.matches { + if _, exists := m.stats[match]; !exists { + err = errors.New(fmt.Sprintf("Parse error for %s : %s", match, name)) + log.Print(err) + continue + } + y, err := lp.New(name, m.tags, map[string]interface{}{"value": int(float64(m.stats[match]) * 1.0e-3)}, time.Now()) + if err == nil { + *out = append(*out, y) + } + } - memUsed := memstats[`MemTotal`] - (memstats[`MemFree`] + memstats[`Buffers`] + memstats[`Cached`]) - m.node["mem_used"] = float64(memUsed) * 1.0e-3 - // In linux-2.5.52 when Memshared was removed - if _, found := memstats[`MemShared`]; found { - m.node["mem_shared"] = float64(memstats[`MemShared`]) * 1.0e-3 + if _, free := m.stats[`MemFree`]; free { + if _, buffers := m.stats[`Buffers`]; buffers { + if _, cached := m.stats[`Cached`]; cached { + memUsed := m.stats[`MemTotal`] - (m.stats[`MemFree`] + m.stats[`Buffers`] + m.stats[`Cached`]) + y, err := lp.New("mem_used", m.tags, map[string]interface{}{"value": int(float64(memUsed) * 1.0e-3)}, time.Now()) + if err == nil { + *out = append(*out, y) + } + } + } + } + if _, found := m.stats[`MemShared`]; found { + y, err := lp.New("mem_shared", m.tags, map[string]interface{}{"value": int(float64(m.stats[`MemShared`]) * 1.0e-3)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } } func (m *MemstatCollector) Close() { + m.init = false return } diff --git a/collectors/metricCollector.go b/collectors/metricCollector.go index 31a4a92..ca076dd 100644 --- a/collectors/metricCollector.go +++ b/collectors/metricCollector.go @@ -1,6 +1,7 @@ package collectors import ( + lp "github.com/influxdata/line-protocol" "io/ioutil" "log" "strconv" @@ -11,48 +12,49 @@ import ( type MetricGetter interface { Name() string Init() error - Read(time.Duration) + Read(time.Duration, *[]lp.MutableMetric) Close() - GetNodeMetric() map[string]interface{} - GetSocketMetrics() map[int]map[string]interface{} - GetCpuMetrics() map[int]map[string]interface{} +// GetNodeMetric() map[string]interface{} +// GetSocketMetrics() map[int]map[string]interface{} +// GetCpuMetrics() map[int]map[string]interface{} } type MetricCollector struct { name string - node map[string]interface{} - sockets map[int]map[string]interface{} - cpus map[int]map[string]interface{} + init bool +// node map[string]interface{} +// sockets map[int]map[string]interface{} +// cpus map[int]map[string]interface{} } func (c *MetricCollector) Name() string { return c.name } -func (c *MetricCollector) GetNodeMetric() map[string]interface{} { - return c.node -} +//func (c *MetricCollector) GetNodeMetric() map[string]interface{} { +// return c.node +//} -func (c *MetricCollector) GetSocketMetrics() map[int]map[string]interface{} { - return c.sockets -} +//func (c *MetricCollector) GetSocketMetrics() map[int]map[string]interface{} { +// return c.sockets +//} -func (c *MetricCollector) GetCpuMetrics() map[int]map[string]interface{} { - return c.cpus -} +//func (c *MetricCollector) GetCpuMetrics() map[int]map[string]interface{} { +// return c.cpus +//} func (c *MetricCollector) setup() error { - slist := SocketList() - clist := CpuList() - c.node = make(map[string]interface{}) - c.sockets = make(map[int]map[string]interface{}, len(slist)) - for _, s := range slist { - c.sockets[s] = make(map[string]interface{}) - } - c.cpus = make(map[int]map[string]interface{}, len(clist)) - for _, s := range clist { - c.cpus[s] = make(map[string]interface{}) - } +// slist := SocketList() +// clist := CpuList() +// c.node = make(map[string]interface{}) +// c.sockets = make(map[int]map[string]interface{}, len(slist)) +// for _, s := range slist { +// c.sockets[s] = make(map[string]interface{}) +// } +// c.cpus = make(map[int]map[string]interface{}, len(clist)) +// for _, s := range clist { +// c.cpus[s] = make(map[string]interface{}) +// } return nil } diff --git a/collectors/netstatMetric.go b/collectors/netstatMetric.go index 5a4b9de..54eae47 100644 --- a/collectors/netstatMetric.go +++ b/collectors/netstatMetric.go @@ -1,7 +1,7 @@ package collectors import ( - "fmt" + lp "github.com/influxdata/line-protocol" "io/ioutil" "log" "strconv" @@ -13,26 +13,33 @@ const NETSTATFILE = `/proc/net/dev` type NetstatCollector struct { MetricCollector + matches map[int]string + tags map[string]string } func (m *NetstatCollector) Init() error { m.name = "NetstatCollector" m.setup() - return nil -} - -func (m *NetstatCollector) Read(interval time.Duration) { - data, err := ioutil.ReadFile(string(NETSTATFILE)) - if err != nil { - log.Print(err.Error()) - return - } - var matches = map[int]string{ + m.tags = map[string]string{"type": "node"} + m.matches = map[int]string{ 1: "bytes_in", 9: "bytes_out", 2: "pkts_in", 10: "pkts_out", } + _, err := ioutil.ReadFile(string(NETSTATFILE)) + if err == nil { + m.init = true + } + return nil +} + +func (m *NetstatCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { + data, err := ioutil.ReadFile(string(NETSTATFILE)) + if err != nil { + log.Print(err.Error()) + return + } lines := strings.Split(string(data), "\n") for _, l := range lines { @@ -44,10 +51,13 @@ func (m *NetstatCollector) Read(interval time.Duration) { if dev == "lo" { continue } - for i, name := range matches { + for i, name := range m.matches { v, err := strconv.ParseInt(f[i], 10, 0) if err == nil { - m.node[fmt.Sprintf("%s_%s", dev, name)] = float64(v) * 1.0e-3 + y, err := lp.New(name, m.tags, map[string]interface{}{"value": int(float64(v) * 1.0e-3)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } } } @@ -55,5 +65,6 @@ func (m *NetstatCollector) Read(interval time.Duration) { } func (m *NetstatCollector) Close() { + m.init = false return } diff --git a/collectors/nvidiaMetric.go b/collectors/nvidiaMetric.go index cfa0435..12312d5 100644 --- a/collectors/nvidiaMetric.go +++ b/collectors/nvidiaMetric.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "github.com/NVIDIA/go-nvml/pkg/nvml" + lp "github.com/influxdata/line-protocol" "log" "time" ) @@ -13,10 +14,21 @@ type NvidiaCollector struct { num_gpus int } +func (m *NvidiaCollector) CatchPanic() error { + + if rerr := recover(); rerr != nil { + log.Print("CatchPanic ", string(rerr.(string))) + err := errors.New(rerr.(string)) + return err + } + return nil +} + func (m *NvidiaCollector) Init() error { m.name = "NvidiaCollector" m.setup() m.num_gpus = 0 + defer m.CatchPanic() ret := nvml.Init() if ret != nvml.SUCCESS { err := errors.New(nvml.ErrorString(ret)) @@ -28,10 +40,11 @@ func (m *NvidiaCollector) Init() error { err := errors.New(nvml.ErrorString(ret)) return err } + m.init = true return nil } -func (m *NvidiaCollector) Read(interval time.Duration) { +func (m *NvidiaCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { for i := 0; i < m.num_gpus; i++ { device, ret := nvml.DeviceGetHandleByIndex(i) @@ -39,113 +52,183 @@ func (m *NvidiaCollector) Read(interval time.Duration) { log.Fatalf("Unable to get device at index %d: %v", i, nvml.ErrorString(ret)) return } - base := fmt.Sprintf("gpu%d", i) + tags := map[string]string{"type": "accelerator", "type-id": fmt.Sprintf("%d", i)} util, ret := nvml.DeviceGetUtilizationRates(device) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_util", base)] = float64(util.Gpu) - m.node[fmt.Sprintf("%s_mem_util", base)] = float64(util.Memory) + y, err := lp.New("util", tags, map[string]interface{}{"value": float64(util.Gpu)}, time.Now()) + if err == nil { + *out = append(*out, y) + } + y, err = lp.New("mem_util", tags, map[string]interface{}{"value": float64(util.Memory)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } meminfo, ret := nvml.DeviceGetMemoryInfo(device) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_mem_total", base)] = float64(meminfo.Total) / (1024 * 1024) - m.node[fmt.Sprintf("%s_fb_memory", base)] = float64(meminfo.Used) / (1024 * 1024) + t := float64(meminfo.Total) / (1024 * 1024) + y, err := lp.New("mem_total", tags, map[string]interface{}{"value": t}, time.Now()) + if err == nil { + *out = append(*out, y) + } + f := float64(meminfo.Used) / (1024 * 1024) + y, err = lp.New("fb_memory", tags, map[string]interface{}{"value": f}, time.Now()) + if err == nil { + *out = append(*out, y) + } } temp, ret := nvml.DeviceGetTemperature(device, nvml.TEMPERATURE_GPU) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_temp", base)] = float64(temp) + y, err := lp.New("temp", tags, map[string]interface{}{"value": float64(temp)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } fan, ret := nvml.DeviceGetFanSpeed(device) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_fan", base)] = float64(fan) + y, err := lp.New("fan", tags, map[string]interface{}{"value": float64(fan)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } _, ecc_pend, ret := nvml.DeviceGetEccMode(device) if ret == nvml.SUCCESS { + var y lp.MutableMetric + var err error switch ecc_pend { case nvml.FEATURE_DISABLED: - m.node[fmt.Sprintf("%s_ecc_mode", base)] = string("OFF") + y, err = lp.New("ecc_mode", tags, map[string]interface{}{"value": string("OFF")}, time.Now()) case nvml.FEATURE_ENABLED: - m.node[fmt.Sprintf("%s_ecc_mode", base)] = string("ON") + y, err = lp.New("ecc_mode", tags, map[string]interface{}{"value": string("ON")}, time.Now()) default: - m.node[fmt.Sprintf("%s_ecc_mode", base)] = string("UNKNOWN") + y, err = lp.New("ecc_mode", tags, map[string]interface{}{"value": string("UNKNOWN")}, time.Now()) + } + if err == nil { + *out = append(*out, y) } } else if ret == nvml.ERROR_NOT_SUPPORTED { - m.node[fmt.Sprintf("%s_ecc_mode", base)] = string("N/A") + y, err := lp.New("ecc_mode", tags, map[string]interface{}{"value": string("N/A")}, time.Now()) + if err == nil { + *out = append(*out, y) + } } pstate, ret := nvml.DeviceGetPerformanceState(device) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_perf_state", base)] = fmt.Sprintf("P%d", int(pstate)) + y, err := lp.New("perf_state", tags, map[string]interface{}{"value": fmt.Sprintf("P%d", int(pstate))}, time.Now()) + if err == nil { + *out = append(*out, y) + } } power, ret := nvml.DeviceGetPowerUsage(device) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_power_usage_report", base)] = float64(power) / 1000 + y, err := lp.New("power_usage_report", tags, map[string]interface{}{"value": float64(power) / 1000}, time.Now()) + if err == nil { + *out = append(*out, y) + } } gclk, ret := nvml.DeviceGetClockInfo(device, nvml.CLOCK_GRAPHICS) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_graphics_clock_report", base)] = float64(gclk) + y, err := lp.New("graphics_clock_report", tags, map[string]interface{}{"value": float64(gclk)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } smclk, ret := nvml.DeviceGetClockInfo(device, nvml.CLOCK_SM) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_sm_clock_report", base)] = float64(smclk) + y, err := lp.New("sm_clock_report", tags, map[string]interface{}{"value": float64(smclk)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } memclk, ret := nvml.DeviceGetClockInfo(device, nvml.CLOCK_MEM) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_mem_clock_report", base)] = float64(memclk) + y, err := lp.New("mem_clock_report", tags, map[string]interface{}{"value": float64(memclk)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } max_gclk, ret := nvml.DeviceGetMaxClockInfo(device, nvml.CLOCK_GRAPHICS) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_max_graphics_clock", base)] = float64(max_gclk) + y, err := lp.New("max_graphics_clock", tags, map[string]interface{}{"value": float64(max_gclk)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } max_smclk, ret := nvml.DeviceGetClockInfo(device, nvml.CLOCK_SM) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_max_sm_clock", base)] = float64(max_smclk) + y, err := lp.New("max_sm_clock", tags, map[string]interface{}{"value": float64(max_smclk)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } max_memclk, ret := nvml.DeviceGetClockInfo(device, nvml.CLOCK_MEM) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_max_mem_clock", base)] = float64(max_memclk) + y, err := lp.New("max_mem_clock", tags, map[string]interface{}{"value": float64(max_memclk)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } ecc_db, ret := nvml.DeviceGetTotalEccErrors(device, 1, 1) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_ecc_db_error", base)] = float64(ecc_db) + y, err := lp.New("ecc_db_error", tags, map[string]interface{}{"value": float64(ecc_db)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } ecc_sb, ret := nvml.DeviceGetTotalEccErrors(device, 0, 1) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_ecc_sb_error", base)] = float64(ecc_sb) + y, err := lp.New("ecc_sb_error", tags, map[string]interface{}{"value": float64(ecc_sb)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } pwr_limit, ret := nvml.DeviceGetPowerManagementLimit(device) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_power_man_limit", base)] = float64(pwr_limit) + y, err := lp.New("power_man_limit", tags, map[string]interface{}{"value": float64(pwr_limit)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } enc_util, _, ret := nvml.DeviceGetEncoderUtilization(device) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_power_man_limit", base)] = float64(enc_util) + y, err := lp.New("encoder_util", tags, map[string]interface{}{"value": float64(enc_util)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } dec_util, _, ret := nvml.DeviceGetDecoderUtilization(device) if ret == nvml.SUCCESS { - m.node[fmt.Sprintf("%s_power_man_limit", base)] = float64(dec_util) + y, err := lp.New("decoder_util", tags, map[string]interface{}{"value": float64(dec_util)}, time.Now()) + if err == nil { + *out = append(*out, y) + } } } } func (m *NvidiaCollector) Close() { - nvml.Shutdown() + if m.init { + nvml.Shutdown() + m.init = false + } return } diff --git a/collectors/topprocsMetric.go b/collectors/topprocsMetric.go index 28f62ba..ffb2065 100644 --- a/collectors/topprocsMetric.go +++ b/collectors/topprocsMetric.go @@ -2,6 +2,7 @@ package collectors import ( "fmt" + lp "github.com/influxdata/line-protocol" "log" "os/exec" "strings" @@ -12,15 +13,23 @@ const NUM_PROCS = 5 type TopProcsCollector struct { MetricCollector + tags map[string]string } func (m *TopProcsCollector) Init() error { m.name = "TopProcsCollector" + m.tags = map[string]string{"type": "node"} m.setup() + command := exec.Command("ps", "-Ao", "comm", "--sort=-pcpu") + command.Wait() + _, err := command.Output() + if err == nil { + m.init = true + } return nil } -func (m *TopProcsCollector) Read(interval time.Duration) { +func (m *TopProcsCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { command := exec.Command("ps", "-Ao", "comm", "--sort=-pcpu") command.Wait() stdout, err := command.Output() @@ -31,10 +40,15 @@ func (m *TopProcsCollector) Read(interval time.Duration) { lines := strings.Split(string(stdout), "\n") for i := 1; i < NUM_PROCS+1; i++ { - m.node[fmt.Sprintf("topproc%d", i)] = lines[i] + name := fmt.Sprintf("topproc%d", i) + y, err := lp.New(name, m.tags, map[string]interface{}{"value": string(lines[i])}, time.Now()) + if err == nil { + *out = append(*out, y) + } } } func (m *TopProcsCollector) Close() { + m.init = false return } diff --git a/config.json b/config.json index 5f5b1c8..d4b8ca8 100644 --- a/config.json +++ b/config.json @@ -3,21 +3,23 @@ "user": "testuser", "password": "testpass", "host": "127.0.0.1", - "port": "8087", + "port": "9090", "database": "testdb", "organization": "testorg", - "type": "influxdb" + "type": "stdout" }, "interval" : 3, "duration" : 1, "collectors": [ - "memstat", - "loadavg", - "netstat", - "ibstat", - "lustrestat", "cpustat", - "topprocs" + "loadavg", + "memstat", + "netstat", + "topprocs", + "lustrestat", + "ibstat", + "nvidia", + "likwid" ], "default_tags": { "cluster": "testcluster" diff --git a/go.mod b/go.mod index 90b6fe0..903ea80 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,10 @@ module github.com/ClusterCockpit/cc-metric-collector go 1.16 require ( + github.com/NVIDIA/go-nvml v0.11.1-0 // indirect github.com/influxdata/influxdb-client-go/v2 v2.2.2 github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 + github.com/nats-io/nats.go v1.10.0 + github.com/nats-io/nkeys v0.1.4 // indirect + github.com/prometheus/client_golang v1.10.0 // indirect ) diff --git a/go.sum b/go.sum index f63eae9..4bd7c8a 100644 --- a/go.sum +++ b/go.sum @@ -1,61 +1,447 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/NVIDIA/go-nvml v0.11.1-0 h1:XHSz3zZKC4NCP2ja1rI7++DXFhA+uDhdYa3MykCTGHY= +github.com/NVIDIA/go-nvml v0.11.1-0/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deepmap/oapi-codegen v1.3.13 h1:9HKGCsdJqE4dnrQ8VerFS0/1ZOJPmAhN+g8xgp8y3K4= github.com/deepmap/oapi-codegen v1.3.13/go.mod h1:WAmG5dWY8/PYHt4vKxlt90NsbHMAOCiteYKZMiIRfOo= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getkin/kin-openapi v0.13.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb-client-go v1.4.0 h1:+KavOkwhLClHFfYcJMHHnTL5CZQhXJzOm5IKHI9BqJk= github.com/influxdata/influxdb-client-go/v2 v2.2.2 h1:O0CGIuIwQafvAxttAJ/VqMKfbWWn2Mt8rbOmaM2Zj4w= github.com/influxdata/influxdb-client-go/v2 v2.2.2/go.mod h1:fa/d1lAdUHxuc1jedx30ZfNG573oQTQmUni3N6pcW+0= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 h1:vilfsDSy7TDxedi9gyBkMvAirat/oRcL0lFdJBf6tdM= github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nats.go v1.10.0 h1:L8qnKaofSfNFbXg0C5F71LdjPRnmQwSsA4ukmkt1TvY= +github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.4 h1:aEsHIssIk6ETN5m2/MD8Y4B2X7FfXrBAUdkyRvbVYzA= +github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg= +github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y= +github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343 h1:00ohfJ4K98s3m6BGUoBd8nyfp4Yl0GoIKvw5abItTjI= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/metric-collector.go b/metric-collector.go index 840837f..a4caa32 100644 --- a/metric-collector.go +++ b/metric-collector.go @@ -7,6 +7,7 @@ import ( "github.com/ClusterCockpit/cc-metric-collector/collectors" "github.com/ClusterCockpit/cc-metric-collector/receivers" "github.com/ClusterCockpit/cc-metric-collector/sinks" + lp "github.com/influxdata/line-protocol" "log" "os" "os/signal" @@ -92,22 +93,22 @@ func SetLogging(logfile string) error { } func CreatePidfile(pidfile string) error { - file, err := os.OpenFile(pidfile, os.O_CREATE|os.O_RDWR, 0600) - if err != nil { - log.Print(err) - return err - } - file.Write([]byte(fmt.Sprintf("%d", os.Getpid()))) - file.Close() - return nil + file, err := os.OpenFile(pidfile, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + log.Print(err) + return err + } + file.Write([]byte(fmt.Sprintf("%d", os.Getpid()))) + file.Close() + return nil } func RemovePidfile(pidfile string) error { - info, err := os.Stat(pidfile) - if !os.IsNotExist(err) && !info.IsDir() { - os.Remove(pidfile) - } - return nil + info, err := os.Stat(pidfile) + if !os.IsNotExist(err) && !info.IsDir() { + os.Remove(pidfile) + } + return nil } // Register an interrupt handler for Ctrl+C and similar. At signal, @@ -150,7 +151,7 @@ func main() { err = CreatePidfile(clicfg["pidfile"]) err = SetLogging(clicfg["logfile"]) if err != nil { - log.Print("Error setting up logging system to ", clicfg["logfile"]) + log.Print("Error setting up logging system to ", clicfg["logfile"], " on ", host) return } @@ -232,21 +233,7 @@ func main() { done := make(chan bool) // Storage for all node metrics - nodeFields := make(map[string]interface{}) - - // Storage for all socket metrics - slist := collectors.SocketList() - socketsFields := make(map[int]map[string]interface{}, len(slist)) - for _, s := range slist { - socketsFields[s] = make(map[string]interface{}) - } - - // Storage for all CPU metrics - clist := collectors.CpuList() - cpuFields := make(map[int]map[string]interface{}, len(clist)) - for _, s := range clist { - cpuFields[s] = make(map[string]interface{}) - } + tmpPoints := make([]lp.MutableMetric, 0) // Start receiver if use_recv { @@ -259,65 +246,24 @@ func main() { case <-done: return case t := <-ticker.C: - // Count how many socket and cpu metrics are returned - scount := 0 - ccount := 0 // Read all collectors are sort the results in the right // storage locations for _, c := range config.Collectors { col := Collectors[c] - col.Read(time.Duration(config.Duration)) + col.Read(time.Duration(config.Duration), &tmpPoints) - for key, val := range col.GetNodeMetric() { - nodeFields[key] = val - } - for sid, socket := range col.GetSocketMetrics() { - for key, val := range socket { - socketsFields[sid][key] = val - scount++ + for { + if len(tmpPoints) == 0 { + break } - } - for cid, cpu := range col.GetCpuMetrics() { - for key, val := range cpu { - cpuFields[cid][key] = val - ccount++ - } - } - } - - // Send out node metrics - if len(nodeFields) > 0 { - nodeTags := map[string]string{"host": host} - for k, v := range config.DefTags { - nodeTags[k] = v - } - sink.Write("node", nodeTags, nodeFields, t) - } - - // Send out socket metrics (if any) - if scount > 0 { - for sid, socket := range socketsFields { - if len(socket) > 0 { - socketTags := map[string]string{"socket": fmt.Sprintf("%d", sid), "host": host} - for k, v := range config.DefTags { - socketTags[k] = v - } - sink.Write("socket", socketTags, socket, t) - } - } - } - - // Send out CPU metrics (if any) - if ccount > 0 { - for cid, cpu := range cpuFields { - if len(cpu) > 0 { - cpuTags := map[string]string{"cpu": fmt.Sprintf("%d", cid), "host": host} - for k, v := range config.DefTags { - cpuTags[k] = v - } - sink.Write("cpu", cpuTags, cpu, t) + p := tmpPoints[0] + for k, v := range config.DefTags { + p.AddTag(k, v) + p.SetTime(t) } + sink.Write(p) + tmpPoints = tmpPoints[1:] } } } diff --git a/receivers/natsReceiver.go b/receivers/natsReceiver.go index bc8effd..32b3e99 100644 --- a/receivers/natsReceiver.go +++ b/receivers/natsReceiver.go @@ -3,7 +3,7 @@ package receivers import ( "errors" s "github.com/ClusterCockpit/cc-metric-collector/sinks" - protocol "github.com/influxdata/line-protocol" + lp "github.com/influxdata/line-protocol" nats "github.com/nats-io/nats.go" "log" "time" @@ -51,15 +51,16 @@ func (r *NatsReceiver) Start() { } func (r *NatsReceiver) _NatsReceive(m *nats.Msg) { - handler := protocol.NewMetricHandler() - parser := protocol.NewParser(handler) + handler := lp.NewMetricHandler() + parser := lp.NewParser(handler) parser.SetTimeFunc(DefaultTime) metrics, err := parser.Parse(m.Data) if err == nil { for _, m := range metrics { - tags := Tags2Map(m) - fields := Fields2Map(m) - r.sink.Write(m.Name(), tags, fields, m.Time()) + y, err := lp.New(m.Name(), Tags2Map(m), Fields2Map(m), m.Time()) + if err == nil { + r.sink.Write(y) + } } } } diff --git a/sinks/influxSink.go b/sinks/influxSink.go index c5d23d4..8c18770 100644 --- a/sinks/influxSink.go +++ b/sinks/influxSink.go @@ -6,8 +6,8 @@ import ( "fmt" influxdb2 "github.com/influxdata/influxdb-client-go/v2" influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api" + lp "github.com/influxdata/line-protocol" "log" - "time" ) type InfluxSink struct { @@ -48,8 +48,16 @@ func (s *InfluxSink) Init(config SinkConfig) error { return s.connect() } -func (s *InfluxSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error { - p := influxdb2.NewPoint(measurement, tags, fields, t) +func (s *InfluxSink) Write(point lp.MutableMetric) error { + var tags map[string]string + var fields map[string]interface{} + for _, t := range point.TagList() { + tags[t.Key] = t.Value + } + for _, f := range point.FieldList() { + fields[f.Key] = f.Value + } + p := influxdb2.NewPoint(point.Name(), tags, fields, point.Time()) err := s.writeApi.WritePoint(context.Background(), p) return err } diff --git a/sinks/metricSink.go b/sinks/metricSink.go index 5c84d8c..c156208 100644 --- a/sinks/metricSink.go +++ b/sinks/metricSink.go @@ -1,7 +1,8 @@ package sinks import ( - "time" + // "time" + lp "github.com/influxdata/line-protocol" ) type SinkConfig struct { @@ -25,6 +26,6 @@ type Sink struct { type SinkFuncs interface { Init(config SinkConfig) error - Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error + Write(point lp.MutableMetric) error Close() } diff --git a/sinks/natsSink.go b/sinks/natsSink.go index 6a938b8..8e5751f 100644 --- a/sinks/natsSink.go +++ b/sinks/natsSink.go @@ -4,7 +4,7 @@ import ( "bytes" "errors" "fmt" - protocol "github.com/influxdata/line-protocol" + lp "github.com/influxdata/line-protocol" nats "github.com/nats-io/nats.go" "log" "time" @@ -13,7 +13,7 @@ import ( type NatsSink struct { Sink client *nats.Conn - encoder *protocol.Encoder + encoder *lp.Encoder buffer *bytes.Buffer } @@ -46,21 +46,29 @@ func (s *NatsSink) Init(config SinkConfig) error { // Setup Influx line protocol s.buffer = &bytes.Buffer{} s.buffer.Grow(1025) - s.encoder = protocol.NewEncoder(s.buffer) + s.encoder = lp.NewEncoder(s.buffer) s.encoder.SetPrecision(time.Second) s.encoder.SetMaxLineBytes(1024) // Setup infos for connection return s.connect() } -func (s *NatsSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error { +func (s *NatsSink) Write(point lp.MutableMetric) error { if s.client != nil { - m, err := protocol.New(measurement, tags, fields, t) - if err != nil { - log.Print(err) - return err - } - _, err = s.encoder.Encode(m) + // var tags map[string]string + // var fields map[string]interface{} + // for _, t := range point.TagList() { + // tags[t.Key] = t.Value + // } + // for _, f := range point.FieldList() { + // fields[f.Key] = f.Value + // } + // m, err := protocol.New(point.Name(), tags, fields, point.Time()) + // if err != nil { + // log.Print(err) + // return err + // } + _, err := s.encoder.Encode(point) if err != nil { log.Print(err) return err diff --git a/sinks/stdoutSink.go b/sinks/stdoutSink.go index 20ec628..a8f7a00 100644 --- a/sinks/stdoutSink.go +++ b/sinks/stdoutSink.go @@ -4,7 +4,8 @@ import ( "fmt" "math" "strings" - "time" + // "time" + lp "github.com/influxdata/line-protocol" ) type StdoutSink struct { @@ -15,26 +16,40 @@ func (s *StdoutSink) Init(config SinkConfig) error { return nil } -func (s *StdoutSink) Write(measurement string, tags map[string]string, fields map[string]interface{}, t time.Time) error { +func (s *StdoutSink) Write(point lp.MutableMetric) error { var tagsstr []string var fieldstr []string - for k, v := range tags { - tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", k, v)) + for _, t := range point.TagList() { + tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", t.Key, t.Value)) } - for k, v := range fields { - switch v.(type) { + for _, f := range point.FieldList() { + switch f.Value.(type) { case float64: - if !math.IsNaN(v.(float64)) { - fieldstr = append(fieldstr, fmt.Sprintf("%s=%v", k, v.(float64))) + if !math.IsNaN(f.Value.(float64)) { + fieldstr = append(fieldstr, fmt.Sprintf("%s=%v", f.Key, f.Value.(float64))) + } else { + fieldstr = append(fieldstr, fmt.Sprintf("%s=0.0", f.Key)) } + case float32: + if !math.IsNaN(float64(f.Value.(float32))) { + fieldstr = append(fieldstr, fmt.Sprintf("%s=%v", f.Key, f.Value.(float32))) + } else { + fieldstr = append(fieldstr, fmt.Sprintf("%s=0.0", f.Key)) + } + case int: + fieldstr = append(fieldstr, fmt.Sprintf("%s=%d", f.Key, f.Value.(int))) + case int64: + fieldstr = append(fieldstr, fmt.Sprintf("%s=%d", f.Key, f.Value.(int64))) case string: - fieldstr = append(fieldstr, fmt.Sprintf("%s=%q", k, v.(string))) + fieldstr = append(fieldstr, fmt.Sprintf("%s=%q", f.Key, f.Value.(string))) + default: + fieldstr = append(fieldstr, fmt.Sprintf("%s=%v", f.Key, f.Value)) } } if len(tagsstr) > 0 { - fmt.Printf("%s,%s %s %d\n", measurement, strings.Join(tagsstr, ","), strings.Join(fieldstr, ","), t.Unix()) + fmt.Printf("%s,%s %s %d\n", point.Name(), strings.Join(tagsstr, ","), strings.Join(fieldstr, ","), point.Time().Unix()) } else { - fmt.Printf("%s %s %d\n", measurement, strings.Join(fieldstr, ","), t.Unix()) + fmt.Printf("%s %s %d\n", point.Name(), strings.Join(fieldstr, ","), point.Time().Unix()) } return nil } From b06572c86588583487dec56e5c8f5629aba5c0d8 Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Mon, 4 Oct 2021 15:46:51 +0200 Subject: [PATCH 02/11] Reset buffer in nats sink --- sinks/natsSink.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sinks/natsSink.go b/sinks/natsSink.go index 8e5751f..efdd3fd 100644 --- a/sinks/natsSink.go +++ b/sinks/natsSink.go @@ -74,7 +74,7 @@ func (s *NatsSink) Write(point lp.MutableMetric) error { return err } s.client.Publish(s.database, s.buffer.Bytes()) - + s.buffer.Reset() } return nil } From 486df91a9df8e173e331e06b6391a6e9a4f24deb Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Mon, 4 Oct 2021 15:47:03 +0200 Subject: [PATCH 03/11] Fix format --- collectors/cpustatMetric.go | 2 +- collectors/infinibandMetric.go | 4 ++-- collectors/loadavgMetric.go | 2 +- collectors/lustreMetric.go | 4 ++-- collectors/memstatMetric.go | 6 +++--- collectors/metricCollector.go | 38 +++++++++++++++++----------------- collectors/netstatMetric.go | 6 +++--- collectors/topprocsMetric.go | 4 ++-- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/collectors/cpustatMetric.go b/collectors/cpustatMetric.go index 9b428b7..7593e8f 100644 --- a/collectors/cpustatMetric.go +++ b/collectors/cpustatMetric.go @@ -69,6 +69,6 @@ func (m *CpustatCollector) Read(interval time.Duration, out *[]lp.MutableMetric) } func (m *CpustatCollector) Close() { - m.init = false + m.init = false return } diff --git a/collectors/infinibandMetric.go b/collectors/infinibandMetric.go index 099bd5b..00c340c 100644 --- a/collectors/infinibandMetric.go +++ b/collectors/infinibandMetric.go @@ -24,7 +24,7 @@ func (m *InfinibandCollector) Init() error { m.tags = map[string]string{"type": "node"} _, err := ioutil.ReadFile(string(LIDFILE)) if err == nil { - m.init = true + m.init = true } return err } @@ -74,6 +74,6 @@ func (m *InfinibandCollector) Read(interval time.Duration, out *[]lp.MutableMetr } func (m *InfinibandCollector) Close() { - m.init = false + m.init = false return } diff --git a/collectors/loadavgMetric.go b/collectors/loadavgMetric.go index 4be7c87..3b91d75 100644 --- a/collectors/loadavgMetric.go +++ b/collectors/loadavgMetric.go @@ -58,6 +58,6 @@ func (m *LoadavgCollector) Read(interval time.Duration, out *[]lp.MutableMetric) } func (m *LoadavgCollector) Close() { - m.init = false + m.init = false return } diff --git a/collectors/lustreMetric.go b/collectors/lustreMetric.go index a4af598..ce52d54 100644 --- a/collectors/lustreMetric.go +++ b/collectors/lustreMetric.go @@ -31,7 +31,7 @@ func (m *LustreCollector) Init() error { "inode_permission": {"inode_permission": 1}} _, err := ioutil.ReadFile(string(LUSTREFILE)) if err == nil { - m.init = true + m.init = true } return err } @@ -65,6 +65,6 @@ func (m *LustreCollector) Read(interval time.Duration, out *[]lp.MutableMetric) } func (m *LustreCollector) Close() { - m.init = false + m.init = false return } diff --git a/collectors/memstatMetric.go b/collectors/memstatMetric.go index 5462de4..d885d44 100644 --- a/collectors/memstatMetric.go +++ b/collectors/memstatMetric.go @@ -36,8 +36,8 @@ func (m *MemstatCollector) Init() error { m.setup() _, err := ioutil.ReadFile(string(MEMSTATFILE)) if err == nil { - m.init = true - } + m.init = true + } return nil } @@ -97,6 +97,6 @@ func (m *MemstatCollector) Read(interval time.Duration, out *[]lp.MutableMetric) } func (m *MemstatCollector) Close() { - m.init = false + m.init = false return } diff --git a/collectors/metricCollector.go b/collectors/metricCollector.go index ca076dd..d05b443 100644 --- a/collectors/metricCollector.go +++ b/collectors/metricCollector.go @@ -14,17 +14,17 @@ type MetricGetter interface { Init() error Read(time.Duration, *[]lp.MutableMetric) Close() -// GetNodeMetric() map[string]interface{} -// GetSocketMetrics() map[int]map[string]interface{} -// GetCpuMetrics() map[int]map[string]interface{} + // GetNodeMetric() map[string]interface{} + // GetSocketMetrics() map[int]map[string]interface{} + // GetCpuMetrics() map[int]map[string]interface{} } type MetricCollector struct { - name string - init bool -// node map[string]interface{} -// sockets map[int]map[string]interface{} -// cpus map[int]map[string]interface{} + name string + init bool + // node map[string]interface{} + // sockets map[int]map[string]interface{} + // cpus map[int]map[string]interface{} } func (c *MetricCollector) Name() string { @@ -44,17 +44,17 @@ func (c *MetricCollector) Name() string { //} func (c *MetricCollector) setup() error { -// slist := SocketList() -// clist := CpuList() -// c.node = make(map[string]interface{}) -// c.sockets = make(map[int]map[string]interface{}, len(slist)) -// for _, s := range slist { -// c.sockets[s] = make(map[string]interface{}) -// } -// c.cpus = make(map[int]map[string]interface{}, len(clist)) -// for _, s := range clist { -// c.cpus[s] = make(map[string]interface{}) -// } + // slist := SocketList() + // clist := CpuList() + // c.node = make(map[string]interface{}) + // c.sockets = make(map[int]map[string]interface{}, len(slist)) + // for _, s := range slist { + // c.sockets[s] = make(map[string]interface{}) + // } + // c.cpus = make(map[int]map[string]interface{}, len(clist)) + // for _, s := range clist { + // c.cpus[s] = make(map[string]interface{}) + // } return nil } diff --git a/collectors/netstatMetric.go b/collectors/netstatMetric.go index 54eae47..b696f74 100644 --- a/collectors/netstatMetric.go +++ b/collectors/netstatMetric.go @@ -29,8 +29,8 @@ func (m *NetstatCollector) Init() error { } _, err := ioutil.ReadFile(string(NETSTATFILE)) if err == nil { - m.init = true - } + m.init = true + } return nil } @@ -65,6 +65,6 @@ func (m *NetstatCollector) Read(interval time.Duration, out *[]lp.MutableMetric) } func (m *NetstatCollector) Close() { - m.init = false + m.init = false return } diff --git a/collectors/topprocsMetric.go b/collectors/topprocsMetric.go index ffb2065..13f547f 100644 --- a/collectors/topprocsMetric.go +++ b/collectors/topprocsMetric.go @@ -24,7 +24,7 @@ func (m *TopProcsCollector) Init() error { command.Wait() _, err := command.Output() if err == nil { - m.init = true + m.init = true } return nil } @@ -49,6 +49,6 @@ func (m *TopProcsCollector) Read(interval time.Duration, out *[]lp.MutableMetric } func (m *TopProcsCollector) Close() { - m.init = false + m.init = false return } From 1762e33c36909af9ddae329d3430a15cf764f49b Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Mon, 4 Oct 2021 16:22:30 +0200 Subject: [PATCH 04/11] Adjust README --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index c785ccf..ec2b263 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A node agent for measuring, processing and forwarding node level metrics part of # Configuration Configuration is implemented using a single json document that is distributed over network and may be persisted as file. -Supported metrics are documented [here](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol.md). +Supported metrics are documented [here](https://github.com/ClusterCockpit/cc-specifications/blob/master/metrics/lineprotocol_alternative.md). ``` json { @@ -86,5 +86,4 @@ The receiver runs as a go routine side-by-side with the timer loop and asynchron - [ ] Use only non-blocking APIs for the sinks - [ ] Collector specific configuration in global JSON file? Changing the configuration inside the Go code is not user-friendly. -- [ ] Metrics in memory domain granularity? From cdc253b569d4d4d39f07508c4eb4559058ef65d3 Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 8 Oct 2021 13:27:15 +0200 Subject: [PATCH 05/11] Return proper error for init() of memstat collector --- collectors/memstatMetric.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collectors/memstatMetric.go b/collectors/memstatMetric.go index d885d44..e163879 100644 --- a/collectors/memstatMetric.go +++ b/collectors/memstatMetric.go @@ -38,7 +38,7 @@ func (m *MemstatCollector) Init() error { if err == nil { m.init = true } - return nil + return err } func (m *MemstatCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { From 7f1694adc7d5fc1e5148415b80ae65168788539c Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 8 Oct 2021 13:27:36 +0200 Subject: [PATCH 06/11] Fix format of infiniband collector --- collectors/infinibandMetric.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/collectors/infinibandMetric.go b/collectors/infinibandMetric.go index a418ae2..8bd6fd3 100644 --- a/collectors/infinibandMetric.go +++ b/collectors/infinibandMetric.go @@ -25,11 +25,11 @@ func (m *InfinibandCollector) Init() error { m.tags = map[string]string{"type": "node"} _, err := ioutil.ReadFile(string(LIDFILE)) if err == nil { - _, err = ioutil.ReadFile(string(PERFQUERY)) - if err == nil { - m.init = true - } - } + _, err = ioutil.ReadFile(string(PERFQUERY)) + if err == nil { + m.init = true + } + } return err } From 46889970711bf575f6d33cf25ee064cba3de50bc Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 8 Oct 2021 13:28:02 +0200 Subject: [PATCH 07/11] Add two helper functions to collector definition --- collectors/metricCollector.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/collectors/metricCollector.go b/collectors/metricCollector.go index d05b443..14c72ec 100644 --- a/collectors/metricCollector.go +++ b/collectors/metricCollector.go @@ -116,3 +116,19 @@ func CpuList() []int { } return cpulist } + +func Tags2Map(metric lp.Metric) map[string]string { + tags := make(map[string]string) + for _, t := range metric.TagList() { + tags[t.Key] = t.Value + } + return tags +} + +func Fields2Map(metric lp.Metric) map[string]interface{} { + fields := make(map[string]interface{}) + for _, f := range metric.FieldList() { + fields[f.Key] = f.Value + } + return fields +} From f74d5dc51ad2f057ec6086f561e7be48d6359938 Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 8 Oct 2021 13:28:44 +0200 Subject: [PATCH 08/11] Reuse parser to avoid overhead in NATS receiver --- receivers/natsReceiver.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/receivers/natsReceiver.go b/receivers/natsReceiver.go index 32b3e99..9d98f00 100644 --- a/receivers/natsReceiver.go +++ b/receivers/natsReceiver.go @@ -11,7 +11,9 @@ import ( type NatsReceiver struct { Receiver - nc *nats.Conn + nc *nats.Conn + handler *lp.MetricHandler + parser *lp.Parser } var DefaultTime = func() time.Time { @@ -42,6 +44,9 @@ func (r *NatsReceiver) Init(config ReceiverConfig, sink s.SinkFuncs) error { log.Print(err) r.nc = nil } + r.handler = lp.NewMetricHandler() + r.parser = lp.NewParser(r.handler) + r.parser.SetTimeFunc(DefaultTime) return err } @@ -51,10 +56,7 @@ func (r *NatsReceiver) Start() { } func (r *NatsReceiver) _NatsReceive(m *nats.Msg) { - handler := lp.NewMetricHandler() - parser := lp.NewParser(handler) - parser.SetTimeFunc(DefaultTime) - metrics, err := parser.Parse(m.Data) + metrics, err := r.parser.Parse(m.Data) if err == nil { for _, m := range metrics { y, err := lp.New(m.Name(), Tags2Map(m), Fields2Map(m), m.Time()) From dc4b8d13c2cbd443a3af82de38e62f986f1e1dac Mon Sep 17 00:00:00 2001 From: Thomas Roehl Date: Fri, 8 Oct 2021 13:29:57 +0200 Subject: [PATCH 09/11] Add collectors for custom commands and /proc/diskstat. Per default add a hostname tag to all measurements --- collectors/customCmdMetric.go | 108 ++++++++++++++++++++++++++++++++++ collectors/diskstatMetric.go | 90 ++++++++++++++++++++++++++++ metric-collector.go | 3 + 3 files changed, 201 insertions(+) create mode 100644 collectors/customCmdMetric.go create mode 100644 collectors/diskstatMetric.go diff --git a/collectors/customCmdMetric.go b/collectors/customCmdMetric.go new file mode 100644 index 0000000..fbb2d87 --- /dev/null +++ b/collectors/customCmdMetric.go @@ -0,0 +1,108 @@ +package collectors + +import ( + "fmt" + lp "github.com/influxdata/line-protocol" + "io/ioutil" + "log" + "os/exec" + "time" +) + +const CUSTOMCMDPATH = `/home/unrz139/Work/cc-metric-collector/collectors/custom` + +type CustomCmdCollector struct { + MetricCollector + handler *lp.MetricHandler + parser *lp.Parser +} + +func (m *CustomCmdCollector) Init() error { + m.name = "CustomCmdCollector" + m.setup() + m.handler = lp.NewMetricHandler() + m.parser = lp.NewParser(m.handler) + m.parser.SetTimeFunc(DefaultTime) + m.init = true + return nil +} + +var DefaultTime = func() time.Time { + return time.Unix(42, 0) +} + +func (m *CustomCmdCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { + files, err := ioutil.ReadDir(string(CUSTOMCMDPATH)) + if err != nil { + log.Print(err) + return + } + for _, file := range files { + // stat, err := os.Stat(file) + // if err != nil { + // log.Print(err) + // continue + // } + // mode := stat.Mode() + // if mode & 0o555 { + path := fmt.Sprintf("%s/%s", string(CUSTOMCMDPATH), file.Name()) + command := exec.Command(path, "") + command.Wait() + stdout, err := command.Output() + if err != nil { + log.Print(err) + continue + } + metrics, err := m.parser.Parse(stdout) + if err != nil { + log.Print(err) + continue + } + for _, m := range metrics { + y, err := lp.New(m.Name(), Tags2Map(m), Fields2Map(m), m.Time()) + if err == nil { + *out = append(*out, y) + } + // switch m.Name() { + // case "node": + // for k, v := range m.FieldList() { + // m.node[k] = float64(v) + // } + // case "socket": + // tlist := m.TagList() + // if id, found := tlist["socket"]; found { + // for k, v := range m.FieldList() { + // m.socket[id][k] = float64(v) + // } + // } + // case "cpu": + // tlist := m.TagList() + // if id, found := tlist["cpu"]; found { + // for k, v := range m.FieldList() { + // m.cpu[id][k] = float64(v) + // } + // } + // case "network": + // tlist := m.TagList() + // if id, found := tlist["device"]; found { + // for k, v := range m.FieldList() { + // m.network[id][k] = float64(v) + // } + // } + // case "accelerator": + // tlist := m.TagList() + // if id, found := tlist["device"]; found { + // for k, v := range m.FieldList() { + // m.accelerator[id][k] = float64(v) + // } + // } + // } + } + // } if file is executable check + } +} + +func (m *CustomCmdCollector) Close() { + m.init = false + return +} diff --git a/collectors/diskstatMetric.go b/collectors/diskstatMetric.go new file mode 100644 index 0000000..fbce9e1 --- /dev/null +++ b/collectors/diskstatMetric.go @@ -0,0 +1,90 @@ +package collectors + +import ( + // "errors" + // "fmt" + lp "github.com/influxdata/line-protocol" + "io/ioutil" + "log" + "strconv" + "strings" + "time" +) + +const DISKSTATFILE = `/proc/diskstats` + +type DiskstatCollector struct { + MetricCollector + matches map[int]string +} + +func (m *DiskstatCollector) Init() error { + m.name = "DiskstatCollector" + m.setup() + // https://www.kernel.org/doc/html/latest/admin-guide/iostats.html + m.matches = map[int]string{ + 3: "reads", + 4: "reads_merged", + 5: "read_sectors", + 6: "read_ms", + 7: "writes", + 8: "writes_merged", + 9: "writes_sectors", + 10: "writes_ms", + 11: "ioops", + 12: "ioops_ms", + 13: "ioops_weighted_ms", + 14: "discards", + 15: "discards_merged", + 16: "discards_sectors", + 17: "discards_ms", + 18: "flushes", + 19: "flushes_ms", + } + _, err := ioutil.ReadFile(string(DISKSTATFILE)) + if err == nil { + m.init = true + } + return err +} + +func (m *DiskstatCollector) Read(interval time.Duration, out *[]lp.MutableMetric) { + + buffer, err := ioutil.ReadFile(string(DISKSTATFILE)) + + if err != nil { + log.Print(err) + return + } + + ll := strings.Split(string(buffer), "\n") + + for _, line := range ll { + if len(line) == 0 { + continue + } + f := strings.Fields(line) + if strings.Contains(f[2], "loop") { + continue + } + tags := map[string]string{ + "device": f[2], + "type": "node", + } + for idx, name := range m.matches { + x, err := strconv.ParseInt(f[idx], 0, 64) + if err == nil { + y, err := lp.New(name, tags, map[string]interface{}{"value": int(x)}, time.Now()) + if err == nil { + *out = append(*out, y) + } + } + } + } + return +} + +func (m *DiskstatCollector) Close() { + m.init = false + return +} diff --git a/metric-collector.go b/metric-collector.go index a4caa32..ebcba52 100644 --- a/metric-collector.go +++ b/metric-collector.go @@ -28,6 +28,8 @@ var Collectors = map[string]collectors.MetricGetter{ "cpustat": &collectors.CpustatCollector{}, "topprocs": &collectors.TopProcsCollector{}, "nvidia": &collectors.NvidiaCollector{}, + "customcmd": &collectors.CustomCmdCollector{}, + "diskstat": &collectors.DiskstatCollector{}, } var Sinks = map[string]sinks.SinkFuncs{ @@ -226,6 +228,7 @@ func main() { } } config.Collectors = tmp + config.DefTags["hostname"] = host // Setup up ticker loop log.Print("Running loop every ", time.Duration(config.Interval)*time.Second) From cdc1811576796e24d1eeac8a64c4b565792f2ece Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Tue, 12 Oct 2021 13:43:58 +0200 Subject: [PATCH 10/11] Add Flush method to sink interface --- metric-collector.go | 4 ++++ sinks/influxSink.go | 9 +++++++-- sinks/metricSink.go | 1 + sinks/natsSink.go | 4 ++++ sinks/stdoutSink.go | 5 +++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/metric-collector.go b/metric-collector.go index ebcba52..ddaa865 100644 --- a/metric-collector.go +++ b/metric-collector.go @@ -269,6 +269,10 @@ func main() { tmpPoints = tmpPoints[1:] } } + + if err := sink.Flush(); err != nil { + log.Printf("sink error: %s\n", err) + } } } }() diff --git a/sinks/influxSink.go b/sinks/influxSink.go index 1b97588..40e681f 100644 --- a/sinks/influxSink.go +++ b/sinks/influxSink.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "errors" "fmt" + influxdb2 "github.com/influxdata/influxdb-client-go/v2" influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api" lp "github.com/influxdata/line-protocol" @@ -57,8 +58,8 @@ func (s *InfluxSink) Init(config SinkConfig) error { } func (s *InfluxSink) Write(point lp.MutableMetric) error { - var tags map[string]string - var fields map[string]interface{} + tags := map[string]string{} + fields := map[string]interface{}{} for _, t := range point.TagList() { tags[t.Key] = t.Value } @@ -70,6 +71,10 @@ func (s *InfluxSink) Write(point lp.MutableMetric) error { return err } +func (s *InfluxSink) Flush() error { + return nil +} + func (s *InfluxSink) Close() { log.Print("Closing InfluxDB connection") s.client.Close() diff --git a/sinks/metricSink.go b/sinks/metricSink.go index 3b70f78..182495a 100644 --- a/sinks/metricSink.go +++ b/sinks/metricSink.go @@ -29,5 +29,6 @@ type Sink struct { type SinkFuncs interface { Init(config SinkConfig) error Write(point lp.MutableMetric) error + Flush() error Close() } diff --git a/sinks/natsSink.go b/sinks/natsSink.go index efdd3fd..0df14f4 100644 --- a/sinks/natsSink.go +++ b/sinks/natsSink.go @@ -79,6 +79,10 @@ func (s *NatsSink) Write(point lp.MutableMetric) error { return nil } +func (s *NatsSink) Flush() error { + return nil +} + func (s *NatsSink) Close() { log.Print("Closing Nats connection") if s.client != nil { diff --git a/sinks/stdoutSink.go b/sinks/stdoutSink.go index a8f7a00..34561e0 100644 --- a/sinks/stdoutSink.go +++ b/sinks/stdoutSink.go @@ -4,6 +4,7 @@ import ( "fmt" "math" "strings" + // "time" lp "github.com/influxdata/line-protocol" ) @@ -54,6 +55,10 @@ func (s *StdoutSink) Write(point lp.MutableMetric) error { return nil } +func (s *StdoutSink) Flush() error { + return nil +} + func (s *StdoutSink) Close() { return } From 485223c5901cdddf0020a2740ba982b69d929482 Mon Sep 17 00:00:00 2001 From: Lou Knauer Date: Tue, 12 Oct 2021 13:44:38 +0200 Subject: [PATCH 11/11] Add new httpSink This sink is compatible with the HTTP API of cc-metric-store. Example config.json section: ``` "sink": { "type": "http", "host": "localhost", "port": "8080", "database": "api/write", "password": "" }, ``` The password/JWT can be omitted. --- README.md | 2 +- metric-collector.go | 1 + sinks/README.md | 1 + sinks/httpSink.go | 68 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 sinks/httpSink.go diff --git a/README.md b/README.md index ec2b263..b1f4aac 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ The `interval` defines how often the metrics should be read and send to the sink All available collectors are listed in the above JSON. A more detailed list can be found in the [README for collectors](./collectors/README.md). -There are currently three sinks supported `influxdb`, `nats` and `stdout`. See [README for sinks](./sinks/README.md). +There are currently four sinks supported `influxdb`, `nats`, `http` and `stdout`. See [README for sinks](./sinks/README.md). In the `default_tags` section, one can define key-value-pairs (only strings) that are added to each sent out metric. This can be useful for cluster names like in the example JSON or information like rank or island for orientation. diff --git a/metric-collector.go b/metric-collector.go index ddaa865..ff67f3c 100644 --- a/metric-collector.go +++ b/metric-collector.go @@ -36,6 +36,7 @@ var Sinks = map[string]sinks.SinkFuncs{ "influxdb": &sinks.InfluxSink{}, "stdout": &sinks.StdoutSink{}, "nats": &sinks.NatsSink{}, + "http": &sinks.HttpSink{}, } var Receivers = map[string]receivers.ReceiverFuncs{ diff --git a/sinks/README.md b/sinks/README.md index 03856ba..43dd802 100644 --- a/sinks/README.md +++ b/sinks/README.md @@ -7,6 +7,7 @@ The base class/configuration is located in `metricSink.go`. * `stdoutSink.go`: Writes all metrics to `stdout` in InfluxDB line protocol. The sink does not use https://github.com/influxdata/line-protocol to reduce the executed code for debugging * `influxSink.go`: Writes all metrics to an InfluxDB database instance using a blocking writer. It uses https://github.com/influxdata/influxdb-client-go . Configuration for the server, port, ssl, password, database name and organisation are in the global configuration file. The 'password' is used for the token and the 'database' for the bucket. It uses the v2 API of Influx. * `natsSink.go`: Sends all metrics to an NATS server using the InfluxDB line protocol as encoding. It uses https://github.com/nats-io/nats.go . Configuration for the server, port, user, password and database name are in the global configuration file. The database name is used as subject for the NATS messages. +* `httpSink.go`: Sends all metrics to an HTTP endpoint `http://:/` using a POST request. The body of the request will consist of lines in the InfluxDB line protocol. In case password is specified, that password is used as a JWT in the 'Authorization' header. # Installation Nothing to do, all sinks are pure Go code diff --git a/sinks/httpSink.go b/sinks/httpSink.go new file mode 100644 index 0000000..e443ceb --- /dev/null +++ b/sinks/httpSink.go @@ -0,0 +1,68 @@ +package sinks + +import ( + "bytes" + "errors" + "fmt" + "net/http" + "time" + + lp "github.com/influxdata/line-protocol" +) + +type HttpSink struct { + Sink + client *http.Client + url, jwt string + encoder *lp.Encoder + buffer *bytes.Buffer +} + +func (s *HttpSink) Init(config SinkConfig) error { + if len(config.Host) == 0 || len(config.Port) == 0 { + return errors.New("`host`, `port` and `database` config options required for TCP sink") + } + + s.client = &http.Client{} + s.url = fmt.Sprintf("http://%s:%s/%s", config.Host, config.Port, config.Database) + s.port = config.Port + s.jwt = config.Password + s.buffer = &bytes.Buffer{} + s.encoder = lp.NewEncoder(s.buffer) + s.encoder.SetPrecision(time.Second) + + return nil +} + +func (s *HttpSink) Write(point lp.MutableMetric) error { + _, err := s.encoder.Encode(point) + return err +} + +func (s *HttpSink) Flush() error { + req, err := http.NewRequest(http.MethodPost, s.url, s.buffer) + if err != nil { + return err + } + + if len(s.jwt) != 0 { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", s.jwt)) + } + + res, err := s.client.Do(req) + s.buffer.Reset() + + if err != nil { + return err + } + + if res.StatusCode != 200 { + return errors.New(res.Status) + } + + return nil +} + +func (s *HttpSink) Close() { + s.client.CloseIdleConnections() +}