mirror of
https://github.com/ClusterCockpit/cc-metric-collector.git
synced 2025-07-19 03:11:41 +02:00
Modularize the whole thing (#16)
* Use channels, add a metric router, split up configuration and use extended version of Influx line protocol internally * Use central timer for collectors and router. Add expressions to router * Add expression to router config * Update entry points * Start with README * Update README for CCMetric * Formatting * Update README.md * Add README for MultiChanTicker * Add README for MultiChanTicker * Update README.md * Add README to metric router * Update main README * Remove SinkEntity type * Update README for sinks * Update go files * Update README for receivers * Update collectors README * Update collectors README * Use seperate page per collector * Fix for tempstat page * Add docs for customcmd collector * Add docs for ipmistat collector * Add docs for topprocs collector * Update customCmdMetric.md * Use seconds when calculating LIKWID metrics * Add IB metrics ib_recv_pkts and ib_xmit_pkts * Drop domain part of host name * Updated to latest stable version of likwid * Define source code dependencies in Makefile * Add GPFS / IBM Spectrum Scale collector * Add vet and staticcheck make targets * Add vet and staticcheck make targets * Avoid go vet warning: struct field tag `json:"..., omitempty"` not compatible with reflect.StructTag.Get: suspicious space in struct tag value struct field tag `json:"...", omitempty` not compatible with reflect.StructTag.Get: key:"value" pairs not separated by spaces * Add sample collector to README.md * Add CPU frequency collector * Avoid staticcheck warning: redundant return statement * Avoid staticcheck warning: unnecessary assignment to the blank identifier * Simplified code * Add CPUFreqCollectorCpuinfo a metric collector to measure the current frequency of the CPUs as obtained from /proc/cpuinfo Only measure on the first hyperthread * Add collector for NFS clients * Move publication of metrics into Flush() for NatsSink * Update GitHub actions * Refactoring * Avoid vet warning: Println arg list ends with redundant newline * Avoid vet warning struct field commands has json tag but is not exported * Avoid vet warning: return copies lock value. * Corrected typo * Refactoring * Add go sources in internal/... * Bad separator in Makefile * Fix Infiniband collector Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com>
This commit is contained in:
Thomas Gruber
GitHub
@@ -1,35 +1,44 @@
|
||||
This folder contains the receivers for the cc-metric-collector.
|
||||
# CCMetric receivers
|
||||
|
||||
# `metricReceiver.go`
|
||||
The base class/configuration is located in `metricReceiver.go`.
|
||||
This folder contains the ReceiveManager and receiver implementations for the cc-metric-collector.
|
||||
|
||||
# Receivers
|
||||
* `natsReceiver.go`: Receives metrics from the Nats transport system in Influx line protocol encoding. The database name is used as subscription subject for the NATS messages. It uses https://github.com/nats-io/nats.go
|
||||
# Configuration
|
||||
|
||||
# Installation
|
||||
Nothing to do, all receivers are pure Go code
|
||||
|
||||
# Receiver configuration
|
||||
The configuration file for the receivers is a list of configurations. The `type` field in each specifies which receiver to initialize.
|
||||
|
||||
```json
|
||||
"receiver": {
|
||||
[
|
||||
{
|
||||
"type": "nats",
|
||||
"address": "nats://my-url"
|
||||
"address": "nats://my-url",
|
||||
"port" : "4222",
|
||||
"database": "testcluster"
|
||||
},
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
## `nats`
|
||||
|
||||
The receiver connects to `address` and `port` and subscribes itself for all messages with topic `database`. The default port is `4222`.
|
||||
## Type `nats`
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "nats",
|
||||
"address": "<nats-URI or hostname>",
|
||||
"port" : "<portnumber>",
|
||||
"database": "<subscribe topic>"
|
||||
}
|
||||
```
|
||||
|
||||
The `nats` receiver subscribes to the topic `database` and listens on `address` and `port` for metrics in the InfluxDB line protocol.
|
||||
|
||||
# Contributing own receivers
|
||||
A receiver contains three functions and is derived from the type `Receiver` (in `metricReceiver.go`):
|
||||
* `Init(config ReceiverConfig) error`
|
||||
* `Start() error`
|
||||
* `Close()`
|
||||
* `Name() string`
|
||||
* `SetSink(sink chan ccMetric.CCMetric)`
|
||||
|
||||
The data structures should be set up in `Init()` like opening a file or server connection. The `Start()` function should either start a go routine or issue some other asynchronous mechanism for receiving metrics. The `Close()` function should tear down anything created in `Init()`.
|
||||
|
||||
Finally, the receiver needs to be registered in the `metric-collector.go`. There is a list of receivers called `Receivers` which is a map (string -> pointer to receiver). Add a new entry with a descriptive name and the new receiver.
|
||||
Finally, the receiver needs to be registered in the `receiveManager.go`. There is a list of receivers called `AvailableReceivers` which is a map (`receiver_type_string` -> `pointer to Receiver interface`). Add a new entry with a descriptive name and the new receiver.
|
||||
|
||||
@@ -2,30 +2,41 @@ package receivers
|
||||
|
||||
import (
|
||||
// "time"
|
||||
s "github.com/ClusterCockpit/cc-metric-collector/sinks"
|
||||
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
|
||||
influx "github.com/influxdata/line-protocol"
|
||||
)
|
||||
|
||||
type ReceiverConfig struct {
|
||||
Addr string `json:"address"`
|
||||
Port string `json:"port"`
|
||||
Database string `json:"database"`
|
||||
Type string `json:"type"`
|
||||
Addr string `json:"address"`
|
||||
Port string `json:"port"`
|
||||
Database string `json:"database"`
|
||||
Organization string `json:"organization,omitempty"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
type Receiver struct {
|
||||
type receiver struct {
|
||||
name string
|
||||
addr string
|
||||
port string
|
||||
database string
|
||||
organization string
|
||||
sink s.SinkFuncs
|
||||
sink chan lp.CCMetric
|
||||
}
|
||||
|
||||
type ReceiverFuncs interface {
|
||||
Init(config ReceiverConfig, sink s.SinkFuncs) error
|
||||
type Receiver interface {
|
||||
Init(config ReceiverConfig) error
|
||||
Start()
|
||||
Close()
|
||||
Name() string
|
||||
SetSink(sink chan lp.CCMetric)
|
||||
}
|
||||
|
||||
func (r *receiver) Name() string {
|
||||
return r.name
|
||||
}
|
||||
|
||||
func (r *receiver) SetSink(sink chan lp.CCMetric) {
|
||||
r.sink = sink
|
||||
}
|
||||
|
||||
func Tags2Map(metric influx.Metric) map[string]string {
|
||||
|
||||
@@ -2,56 +2,68 @@ package receivers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
s "github.com/ClusterCockpit/cc-metric-collector/sinks"
|
||||
lp "github.com/influxdata/line-protocol"
|
||||
"fmt"
|
||||
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
|
||||
influx "github.com/influxdata/line-protocol"
|
||||
nats "github.com/nats-io/nats.go"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
type NatsReceiverConfig struct {
|
||||
Addr string `json:"address"`
|
||||
Port string `json:"port"`
|
||||
Database string `json:"database"`
|
||||
}
|
||||
|
||||
type NatsReceiver struct {
|
||||
Receiver
|
||||
receiver
|
||||
nc *nats.Conn
|
||||
handler *lp.MetricHandler
|
||||
parser *lp.Parser
|
||||
handler *influx.MetricHandler
|
||||
parser *influx.Parser
|
||||
meta map[string]string
|
||||
config ReceiverConfig
|
||||
}
|
||||
|
||||
var DefaultTime = func() time.Time {
|
||||
return time.Unix(42, 0)
|
||||
}
|
||||
|
||||
func (r *NatsReceiver) Init(config ReceiverConfig, sink s.SinkFuncs) error {
|
||||
if len(config.Addr) == 0 ||
|
||||
len(config.Port) == 0 ||
|
||||
len(config.Database) == 0 {
|
||||
func (r *NatsReceiver) Init(config ReceiverConfig) error {
|
||||
r.name = "NatsReceiver"
|
||||
r.config = config
|
||||
if len(r.config.Addr) == 0 ||
|
||||
len(r.config.Port) == 0 ||
|
||||
len(r.config.Database) == 0 {
|
||||
return errors.New("Not all configuration variables set required by NatsReceiver")
|
||||
}
|
||||
r.addr = config.Addr
|
||||
r.meta = map[string]string{"source": r.name}
|
||||
r.addr = r.config.Addr
|
||||
if len(r.addr) == 0 {
|
||||
r.addr = nats.DefaultURL
|
||||
}
|
||||
r.port = config.Port
|
||||
r.port = r.config.Port
|
||||
if len(r.port) == 0 {
|
||||
r.port = "4222"
|
||||
}
|
||||
log.Print("Init NATS Receiver")
|
||||
nc, err := nats.Connect(r.addr)
|
||||
log.Print("[NatsReceiver] INIT")
|
||||
uri := fmt.Sprintf("%s:%s", r.addr, r.port)
|
||||
nc, err := nats.Connect(uri)
|
||||
if err == nil {
|
||||
r.database = config.Database
|
||||
r.sink = sink
|
||||
r.database = r.config.Database
|
||||
r.nc = nc
|
||||
} else {
|
||||
log.Print(err)
|
||||
r.nc = nil
|
||||
return err
|
||||
}
|
||||
r.handler = lp.NewMetricHandler()
|
||||
r.parser = lp.NewParser(r.handler)
|
||||
r.handler = influx.NewMetricHandler()
|
||||
r.parser = influx.NewParser(r.handler)
|
||||
r.parser.SetTimeFunc(DefaultTime)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *NatsReceiver) Start() {
|
||||
log.Print("Start NATS Receiver")
|
||||
log.Print("[NatsReceiver] START")
|
||||
r.nc.Subscribe(r.database, r._NatsReceive)
|
||||
}
|
||||
|
||||
@@ -59,9 +71,13 @@ func (r *NatsReceiver) _NatsReceive(m *nats.Msg) {
|
||||
metrics, err := r.parser.Parse(m.Data)
|
||||
if err == nil {
|
||||
for _, m := range metrics {
|
||||
y, err := lp.New(m.Name(), Tags2Map(m), Fields2Map(m), m.Time())
|
||||
if err == nil {
|
||||
r.sink.Write(y)
|
||||
y := lp.FromInfluxMetric(m)
|
||||
for k, v := range r.meta {
|
||||
y.AddMeta(k, v)
|
||||
}
|
||||
//y, err := lp.New(m.Name(), Tags2Map(m), r.meta, Fields2Map(m), m.Time())
|
||||
if r.sink != nil {
|
||||
r.sink <- y
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -69,7 +85,7 @@ func (r *NatsReceiver) _NatsReceive(m *nats.Msg) {
|
||||
|
||||
func (r *NatsReceiver) Close() {
|
||||
if r.nc != nil {
|
||||
log.Print("Close NATS Receiver")
|
||||
log.Print("[NatsReceiver] CLOSE")
|
||||
r.nc.Close()
|
||||
}
|
||||
}
|
||||
|
||||
153
receivers/receiveManager.go
Normal file
153
receivers/receiveManager.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package receivers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
|
||||
"log"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var AvailableReceivers = map[string]Receiver{
|
||||
"nats": &NatsReceiver{},
|
||||
}
|
||||
|
||||
type receiveManager struct {
|
||||
inputs []Receiver
|
||||
output chan lp.CCMetric
|
||||
done chan bool
|
||||
wg *sync.WaitGroup
|
||||
config []ReceiverConfig
|
||||
}
|
||||
|
||||
type ReceiveManager interface {
|
||||
Init(wg *sync.WaitGroup, receiverConfigFile string) error
|
||||
AddInput(rawConfig json.RawMessage) error
|
||||
AddOutput(output chan lp.CCMetric)
|
||||
Start()
|
||||
Close()
|
||||
}
|
||||
|
||||
func (rm *receiveManager) Init(wg *sync.WaitGroup, receiverConfigFile string) error {
|
||||
rm.inputs = make([]Receiver, 0)
|
||||
rm.output = nil
|
||||
rm.done = make(chan bool)
|
||||
rm.wg = wg
|
||||
rm.config = make([]ReceiverConfig, 0)
|
||||
configFile, err := os.Open(receiverConfigFile)
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
return err
|
||||
}
|
||||
defer configFile.Close()
|
||||
jsonParser := json.NewDecoder(configFile)
|
||||
var rawConfigs []json.RawMessage
|
||||
err = jsonParser.Decode(&rawConfigs)
|
||||
if err != nil {
|
||||
log.Print(err.Error())
|
||||
return err
|
||||
}
|
||||
for _, raw := range rawConfigs {
|
||||
log.Print("[ReceiveManager] ", string(raw))
|
||||
rm.AddInput(raw)
|
||||
// if _, found := AvailableReceivers[k.Type]; !found {
|
||||
// log.Print("[ReceiveManager] SKIP Config specifies unknown receiver 'type': ", k.Type)
|
||||
// continue
|
||||
// }
|
||||
// r := AvailableReceivers[k.Type]
|
||||
// err = r.Init(k)
|
||||
// if err != nil {
|
||||
// log.Print("[ReceiveManager] SKIP Receiver ", k.Type, " cannot be initialized: ", err.Error())
|
||||
// continue
|
||||
// }
|
||||
// rm.inputs = append(rm.inputs, r)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rm *receiveManager) Start() {
|
||||
rm.wg.Add(1)
|
||||
|
||||
for _, r := range rm.inputs {
|
||||
log.Print("[ReceiveManager] START ", r.Name())
|
||||
r.Start()
|
||||
}
|
||||
log.Print("[ReceiveManager] STARTED\n")
|
||||
// go func() {
|
||||
// for {
|
||||
//ReceiveManagerLoop:
|
||||
// select {
|
||||
// case <- rm.done:
|
||||
// log.Print("ReceiveManager done\n")
|
||||
// rm.wg.Done()
|
||||
// break ReceiveManagerLoop
|
||||
// default:
|
||||
// for _, c := range rm.inputs {
|
||||
//ReceiveManagerInputLoop:
|
||||
// select {
|
||||
// case <- rm.done:
|
||||
// log.Print("ReceiveManager done\n")
|
||||
// rm.wg.Done()
|
||||
// break ReceiveManagerInputLoop
|
||||
// case p := <- c:
|
||||
// log.Print("ReceiveManager: ", p)
|
||||
// rm.output <- p
|
||||
// default:
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }()
|
||||
// for _, r := range rm.inputs {
|
||||
// r.Close()
|
||||
// }
|
||||
}
|
||||
|
||||
func (rm *receiveManager) AddInput(rawConfig json.RawMessage) error {
|
||||
var config ReceiverConfig
|
||||
err := json.Unmarshal(rawConfig, &config)
|
||||
if err != nil {
|
||||
log.Print("[ReceiveManager] SKIP ", config.Type, " JSON config error: ", err.Error())
|
||||
log.Print(err.Error())
|
||||
return err
|
||||
}
|
||||
if _, found := AvailableReceivers[config.Type]; !found {
|
||||
log.Print("[ReceiveManager] SKIP ", config.Type, " unknown receiver: ", err.Error())
|
||||
return err
|
||||
}
|
||||
r := AvailableReceivers[config.Type]
|
||||
err = r.Init(config)
|
||||
if err != nil {
|
||||
log.Print("[ReceiveManager] SKIP ", r.Name(), " initialization failed: ", err.Error())
|
||||
return err
|
||||
}
|
||||
rm.inputs = append(rm.inputs, r)
|
||||
rm.config = append(rm.config, config)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rm *receiveManager) AddOutput(output chan lp.CCMetric) {
|
||||
rm.output = output
|
||||
for _, r := range rm.inputs {
|
||||
r.SetSink(rm.output)
|
||||
}
|
||||
}
|
||||
|
||||
func (rm *receiveManager) Close() {
|
||||
for _, r := range rm.inputs {
|
||||
log.Print("[ReceiveManager] CLOSE ", r.Name())
|
||||
r.Close()
|
||||
}
|
||||
rm.wg.Done()
|
||||
log.Print("[ReceiveManager] CLOSE\n")
|
||||
log.Print("[ReceiveManager] EXIT\n")
|
||||
}
|
||||
|
||||
func New(wg *sync.WaitGroup, receiverConfigFile string) (ReceiveManager, error) {
|
||||
r := &receiveManager{}
|
||||
err := r.Init(wg, receiverConfigFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
Reference in New Issue
Block a user