mirror of
https://github.com/ClusterCockpit/cc-metric-collector.git
synced 2025-08-22 18:33:00 +02:00
Merge develop branch into main (#123)
* Add cpu_used (all-cpu_idle) to CpustatCollector * Update cc-metric-collector.init * Allow selection of timestamp precision in HttpSink * Add comment about precision requirement for cc-metric-store * Fix for API changes in gofish@v0.15.0 * Update requirements to latest version * Read sensors through redfish * Update golang toolchain to 1.21 * Remove stray error check * Update main config in configuration.md * Update Release action to use golang 1.22 stable release, no golang RPMs anymore * Update runonce action to use golang 1.22 stable release, no golang RPMs anymore * Update README.md Use right JSON type in configuration * Update sink's README * Test whether ipmitool or ipmi-sensors can be executed without errors * Little fixes to the prometheus sink (#115) * Add uint64 to float64 cast option * Add prometheus sink to the list of available sinks * Add aggregated counters by gpu for nvlink errors --------- Co-authored-by: Michael Schwarz <schwarz@uni-paderborn.de> * Ccmessage migration (#119) * Add cpu_used (all-cpu_idle) to CpustatCollector * Update cc-metric-collector.init * Allow selection of timestamp precision in HttpSink * Add comment about precision requirement for cc-metric-store * Fix for API changes in gofish@v0.15.0 * Update requirements to latest version * Read sensors through redfish * Update golang toolchain to 1.21 * Remove stray error check * Update main config in configuration.md * Update Release action to use golang 1.22 stable release, no golang RPMs anymore * Update runonce action to use golang 1.22 stable release, no golang RPMs anymore * Switch to CCMessage for all files. --------- Co-authored-by: Holger Obermaier <Holger.Obermaier@kit.edu> Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> * Switch to ccmessage also for latest additions in nvidiaMetric * New Message processor (#118) * Add cpu_used (all-cpu_idle) to CpustatCollector * Update cc-metric-collector.init * Allow selection of timestamp precision in HttpSink * Add comment about precision requirement for cc-metric-store * Fix for API changes in gofish@v0.15.0 * Update requirements to latest version * Read sensors through redfish * Update golang toolchain to 1.21 * Remove stray error check * Update main config in configuration.md * Update Release action to use golang 1.22 stable release, no golang RPMs anymore * Update runonce action to use golang 1.22 stable release, no golang RPMs anymore * New message processor to check whether a message should be dropped or manipulate it in flight * Create a copy of message before manipulation --------- Co-authored-by: Holger Obermaier <Holger.Obermaier@kit.edu> Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> * Update collector's Makefile and go.mod/sum files * Use message processor in router, all sinks and all receivers * Add support for credential file (NKEY) to NATS sink and receiver * Fix JSON keys in message processor configuration * Update docs for message processor, router and the default router config file * Add link to expr syntax and fix regex matching docs * Update sample collectors * Minor style change in collector manager * Some helpers for ccTopology * LIKWID collector: write log owner change only once * Fix for metrics without units and reduce debugging messages for messageProcessor * Use shorted hostname for hostname added by router * Define default port for NATS * CPUstat collector: only add unit for applicable metrics * Add precision option to all sinks using Influx's encoder * Add message processor to all sink documentation * Add units to documentation of cpustat collector --------- Co-authored-by: Holger Obermaier <Holger.Obermaier@kit.edu> Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> Co-authored-by: oscarminus <me@oscarminus.de> Co-authored-by: Michael Schwarz <schwarz@uni-paderborn.de>
This commit is contained in:
Thomas Gruber
GitHub
@@ -10,15 +10,16 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
|
||||
cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
|
||||
lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
|
||||
mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
|
||||
influx "github.com/influxdata/line-protocol/v2/lineprotocol"
|
||||
)
|
||||
|
||||
const HTTP_RECEIVER_PORT = "8080"
|
||||
|
||||
type HttpReceiverConfig struct {
|
||||
Type string `json:"type"`
|
||||
defaultReceiverConfig
|
||||
Addr string `json:"address"`
|
||||
Port string `json:"port"`
|
||||
Path string `json:"path"`
|
||||
@@ -39,7 +40,7 @@ type HttpReceiverConfig struct {
|
||||
|
||||
type HttpReceiver struct {
|
||||
receiver
|
||||
meta map[string]string
|
||||
//meta map[string]string
|
||||
config HttpReceiverConfig
|
||||
server *http.Server
|
||||
wg sync.WaitGroup
|
||||
@@ -85,8 +86,20 @@ func (r *HttpReceiver) Init(name string, config json.RawMessage) error {
|
||||
if r.config.useBasicAuth && len(r.config.Password) == 0 {
|
||||
return errors.New("basic authentication requires password")
|
||||
}
|
||||
msgp, err := mp.NewMessageProcessor()
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialization of message processor failed: %v", err.Error())
|
||||
}
|
||||
r.mp = msgp
|
||||
if len(r.config.MessageProcessor) > 0 {
|
||||
err = r.mp.FromConfigJSON(r.config.MessageProcessor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
|
||||
}
|
||||
}
|
||||
r.mp.AddAddMetaByCondition("true", "source", r.name)
|
||||
|
||||
r.meta = map[string]string{"source": r.name}
|
||||
//r.meta = map[string]string{"source": r.name}
|
||||
p := r.config.Path
|
||||
if !strings.HasPrefix(p, "/") {
|
||||
p = "/" + p
|
||||
@@ -137,80 +150,82 @@ func (r *HttpReceiver) ServerHttp(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if r.sink != nil {
|
||||
d := influx.NewDecoder(req.Body)
|
||||
for d.Next() {
|
||||
|
||||
d := influx.NewDecoder(req.Body)
|
||||
for d.Next() {
|
||||
|
||||
// Decode measurement name
|
||||
measurement, err := d.Measurement()
|
||||
if err != nil {
|
||||
msg := "ServerHttp: Failed to decode measurement: " + err.Error()
|
||||
cclog.ComponentError(r.name, msg)
|
||||
http.Error(w, msg, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Decode tags
|
||||
tags := make(map[string]string)
|
||||
for {
|
||||
key, value, err := d.NextTag()
|
||||
// Decode measurement name
|
||||
measurement, err := d.Measurement()
|
||||
if err != nil {
|
||||
msg := "ServerHttp: Failed to decode tag: " + err.Error()
|
||||
msg := "ServerHttp: Failed to decode measurement: " + err.Error()
|
||||
cclog.ComponentError(r.name, msg)
|
||||
http.Error(w, msg, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if key == nil {
|
||||
break
|
||||
}
|
||||
tags[string(key)] = string(value)
|
||||
}
|
||||
|
||||
// Decode fields
|
||||
fields := make(map[string]interface{})
|
||||
for {
|
||||
key, value, err := d.NextField()
|
||||
// Decode tags
|
||||
tags := make(map[string]string)
|
||||
for {
|
||||
key, value, err := d.NextTag()
|
||||
if err != nil {
|
||||
msg := "ServerHttp: Failed to decode tag: " + err.Error()
|
||||
cclog.ComponentError(r.name, msg)
|
||||
http.Error(w, msg, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if key == nil {
|
||||
break
|
||||
}
|
||||
tags[string(key)] = string(value)
|
||||
}
|
||||
|
||||
// Decode fields
|
||||
fields := make(map[string]interface{})
|
||||
for {
|
||||
key, value, err := d.NextField()
|
||||
if err != nil {
|
||||
msg := "ServerHttp: Failed to decode field: " + err.Error()
|
||||
cclog.ComponentError(r.name, msg)
|
||||
http.Error(w, msg, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if key == nil {
|
||||
break
|
||||
}
|
||||
fields[string(key)] = value.Interface()
|
||||
}
|
||||
|
||||
// Decode time stamp
|
||||
t, err := d.Time(influx.Nanosecond, time.Time{})
|
||||
if err != nil {
|
||||
msg := "ServerHttp: Failed to decode field: " + err.Error()
|
||||
msg := "ServerHttp: Failed to decode time stamp: " + err.Error()
|
||||
cclog.ComponentError(r.name, msg)
|
||||
http.Error(w, msg, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if key == nil {
|
||||
break
|
||||
}
|
||||
fields[string(key)] = value.Interface()
|
||||
}
|
||||
|
||||
// Decode time stamp
|
||||
t, err := d.Time(influx.Nanosecond, time.Time{})
|
||||
y, _ := lp.NewMessage(
|
||||
string(measurement),
|
||||
tags,
|
||||
nil,
|
||||
fields,
|
||||
t,
|
||||
)
|
||||
|
||||
m, err := r.mp.ProcessMessage(y)
|
||||
if err == nil && m != nil {
|
||||
r.sink <- m
|
||||
}
|
||||
|
||||
}
|
||||
// Check for IO errors
|
||||
err := d.Err()
|
||||
if err != nil {
|
||||
msg := "ServerHttp: Failed to decode time stamp: " + err.Error()
|
||||
msg := "ServerHttp: Failed to decode: " + err.Error()
|
||||
cclog.ComponentError(r.name, msg)
|
||||
http.Error(w, msg, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
y, _ := lp.New(
|
||||
string(measurement),
|
||||
tags,
|
||||
r.meta,
|
||||
fields,
|
||||
t,
|
||||
)
|
||||
|
||||
if r.sink != nil {
|
||||
r.sink <- y
|
||||
}
|
||||
}
|
||||
|
||||
// Check for IO errors
|
||||
err := d.Err()
|
||||
if err != nil {
|
||||
msg := "ServerHttp: Failed to decode: " + err.Error()
|
||||
cclog.ComponentError(r.name, msg)
|
||||
http.Error(w, msg, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
Reference in New Issue
Block a user