Merge develop branch into main (#123)

* Add cpu_used (all-cpu_idle) to CpustatCollector * Update cc-metric-collector.init * Allow selection of timestamp precision in HttpSink * Add comment about precision requirement for cc-metric-store * Fix for API changes in gofish@v0.15.0 * Update requirements to latest version * Read sensors through redfish * Update golang toolchain to 1.21 * Remove stray error check * Update main config in configuration.md * Update Release action to use golang 1.22 stable release, no golang RPMs anymore * Update runonce action to use golang 1.22 stable release, no golang RPMs anymore * Update README.md Use right JSON type in configuration * Update sink's README * Test whether ipmitool or ipmi-sensors can be executed without errors * Little fixes to the prometheus sink (#115) * Add uint64 to float64 cast option * Add prometheus sink to the list of available sinks * Add aggregated counters by gpu for nvlink errors --------- Co-authored-by: Michael Schwarz <schwarz@uni-paderborn.de> * Ccmessage migration (#119) * Add cpu_used (all-cpu_idle) to CpustatCollector * Update cc-metric-collector.init * Allow selection of timestamp precision in HttpSink * Add comment about precision requirement for cc-metric-store * Fix for API changes in gofish@v0.15.0 * Update requirements to latest version * Read sensors through redfish * Update golang toolchain to 1.21 * Remove stray error check * Update main config in configuration.md * Update Release action to use golang 1.22 stable release, no golang RPMs anymore * Update runonce action to use golang 1.22 stable release, no golang RPMs anymore * Switch to CCMessage for all files. --------- Co-authored-by: Holger Obermaier <Holger.Obermaier@kit.edu> Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> * Switch to ccmessage also for latest additions in nvidiaMetric * New Message processor (#118) * Add cpu_used (all-cpu_idle) to CpustatCollector * Update cc-metric-collector.init * Allow selection of timestamp precision in HttpSink * Add comment about precision requirement for cc-metric-store * Fix for API changes in gofish@v0.15.0 * Update requirements to latest version * Read sensors through redfish * Update golang toolchain to 1.21 * Remove stray error check * Update main config in configuration.md * Update Release action to use golang 1.22 stable release, no golang RPMs anymore * Update runonce action to use golang 1.22 stable release, no golang RPMs anymore * New message processor to check whether a message should be dropped or manipulate it in flight * Create a copy of message before manipulation --------- Co-authored-by: Holger Obermaier <Holger.Obermaier@kit.edu> Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> * Update collector's Makefile and go.mod/sum files * Use message processor in router, all sinks and all receivers * Add support for credential file (NKEY) to NATS sink and receiver * Fix JSON keys in message processor configuration * Update docs for message processor, router and the default router config file * Add link to expr syntax and fix regex matching docs * Update sample collectors * Minor style change in collector manager * Some helpers for ccTopology * LIKWID collector: write log owner change only once * Fix for metrics without units and reduce debugging messages for messageProcessor * Use shorted hostname for hostname added by router * Define default port for NATS * CPUstat collector: only add unit for applicable metrics * Add precision option to all sinks using Influx's encoder * Add message processor to all sink documentation * Add units to documentation of cpustat collector --------- Co-authored-by: Holger Obermaier <Holger.Obermaier@kit.edu> Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> Co-authored-by: oscarminus <me@oscarminus.de> Co-authored-by: Michael Schwarz <schwarz@uni-paderborn.de>
2025-07-28 23:56:08 +02:00 · 2024-12-19 23:00:14 +01:00
parent 21646e1edf
commit 7840de7b82
74 changed files with 1902 additions and 1017 deletions
--- a/receivers/httpReceiver.go
+++ b/receivers/httpReceiver.go
@@ -10,15 +10,16 @@ import (
 	"sync"
 	"time"

+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
 	influx "github.com/influxdata/line-protocol/v2/lineprotocol"
 )

 const HTTP_RECEIVER_PORT = "8080"

 type HttpReceiverConfig struct {
-	Type string `json:"type"`
+	defaultReceiverConfig
 	Addr string `json:"address"`
 	Port string `json:"port"`
 	Path string `json:"path"`
@@ -39,7 +40,7 @@ type HttpReceiverConfig struct {

 type HttpReceiver struct {
 	receiver
-	meta   map[string]string
+	//meta   map[string]string
 	config HttpReceiverConfig
 	server *http.Server
 	wg     sync.WaitGroup
@@ -85,8 +86,20 @@ func (r *HttpReceiver) Init(name string, config json.RawMessage) error {
 	if r.config.useBasicAuth && len(r.config.Password) == 0 {
 		return errors.New("basic authentication requires password")
 	}
+	msgp, err := mp.NewMessageProcessor()
+	if err != nil {
+		return fmt.Errorf("initialization of message processor failed: %v", err.Error())
+	}
+	r.mp = msgp
+	if len(r.config.MessageProcessor) > 0 {
+		err = r.mp.FromConfigJSON(r.config.MessageProcessor)
+		if err != nil {
+			return fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+		}
+	}
+	r.mp.AddAddMetaByCondition("true", "source", r.name)

-	r.meta = map[string]string{"source": r.name}
+	//r.meta = map[string]string{"source": r.name}
 	p := r.config.Path
 	if !strings.HasPrefix(p, "/") {
 		p = "/" + p
@@ -137,80 +150,82 @@ func (r *HttpReceiver) ServerHttp(w http.ResponseWriter, req *http.Request) {
 			return
 		}
 	}
+	if r.sink != nil {
+		d := influx.NewDecoder(req.Body)
+		for d.Next() {

-	d := influx.NewDecoder(req.Body)
-	for d.Next() {
-
-		// Decode measurement name
-		measurement, err := d.Measurement()
-		if err != nil {
-			msg := "ServerHttp: Failed to decode measurement: " + err.Error()
-			cclog.ComponentError(r.name, msg)
-			http.Error(w, msg, http.StatusInternalServerError)
-			return
-		}
-
-		// Decode tags
-		tags := make(map[string]string)
-		for {
-			key, value, err := d.NextTag()
+			// Decode measurement name
+			measurement, err := d.Measurement()
 			if err != nil {
-				msg := "ServerHttp: Failed to decode tag: " + err.Error()
+				msg := "ServerHttp: Failed to decode measurement: " + err.Error()
 				cclog.ComponentError(r.name, msg)
 				http.Error(w, msg, http.StatusInternalServerError)
 				return
 			}
-			if key == nil {
-				break
-			}
-			tags[string(key)] = string(value)
-		}

-		// Decode fields
-		fields := make(map[string]interface{})
-		for {
-			key, value, err := d.NextField()
+			// Decode tags
+			tags := make(map[string]string)
+			for {
+				key, value, err := d.NextTag()
+				if err != nil {
+					msg := "ServerHttp: Failed to decode tag: " + err.Error()
+					cclog.ComponentError(r.name, msg)
+					http.Error(w, msg, http.StatusInternalServerError)
+					return
+				}
+				if key == nil {
+					break
+				}
+				tags[string(key)] = string(value)
+			}
+
+			// Decode fields
+			fields := make(map[string]interface{})
+			for {
+				key, value, err := d.NextField()
+				if err != nil {
+					msg := "ServerHttp: Failed to decode field: " + err.Error()
+					cclog.ComponentError(r.name, msg)
+					http.Error(w, msg, http.StatusInternalServerError)
+					return
+				}
+				if key == nil {
+					break
+				}
+				fields[string(key)] = value.Interface()
+			}
+
+			// Decode time stamp
+			t, err := d.Time(influx.Nanosecond, time.Time{})
 			if err != nil {
-				msg := "ServerHttp: Failed to decode field: " + err.Error()
+				msg := "ServerHttp: Failed to decode time stamp: " + err.Error()
 				cclog.ComponentError(r.name, msg)
 				http.Error(w, msg, http.StatusInternalServerError)
 				return
 			}
-			if key == nil {
-				break
-			}
-			fields[string(key)] = value.Interface()
-		}

-		// Decode time stamp
-		t, err := d.Time(influx.Nanosecond, time.Time{})
+			y, _ := lp.NewMessage(
+				string(measurement),
+				tags,
+				nil,
+				fields,
+				t,
+			)
+
+			m, err := r.mp.ProcessMessage(y)
+			if err == nil && m != nil {
+				r.sink <- m
+			}
+
+		}
+		// Check for IO errors
+		err := d.Err()
 		if err != nil {
-			msg := "ServerHttp: Failed to decode time stamp: " + err.Error()
+			msg := "ServerHttp: Failed to decode: " + err.Error()
 			cclog.ComponentError(r.name, msg)
 			http.Error(w, msg, http.StatusInternalServerError)
 			return
 		}
-
-		y, _ := lp.New(
-			string(measurement),
-			tags,
-			r.meta,
-			fields,
-			t,
-		)
-
-		if r.sink != nil {
-			r.sink <- y
-		}
-	}
-
-	// Check for IO errors
-	err := d.Err()
-	if err != nil {
-		msg := "ServerHttp: Failed to decode: " + err.Error()
-		cclog.ComponentError(r.name, msg)
-		http.Error(w, msg, http.StatusInternalServerError)
-		return
 	}

 	w.WriteHeader(http.StatusOK)
--- a/receivers/ipmiReceiver.go
+++ b/receivers/ipmiReceiver.go
@@ -13,9 +13,10 @@ import (
 	"sync"
 	"time"

+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 	"github.com/ClusterCockpit/cc-metric-collector/pkg/hostlist"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
 )

 type IPMIReceiverClientConfig struct {
@@ -31,11 +32,13 @@ type IPMIReceiverClientConfig struct {
 	Password         string            // Password to use for authentication
 	CLIOptions       []string          // Additional command line options for ipmi-sensors
 	isExcluded       map[string]bool   // is metric excluded
+	mp               mp.MessageProcessor
 }

 type IPMIReceiver struct {
 	receiver
 	config struct {
+		defaultReceiverConfig
 		Interval time.Duration

 		// Client config for each IPMI hosts
@@ -43,10 +46,11 @@ type IPMIReceiver struct {
 	}

 	// Storage for static information
-	meta map[string]string
+	//meta map[string]string

 	done chan bool      // channel to finish / stop IPMI receiver
 	wg   sync.WaitGroup // wait group for IPMI receiver
+	mp   mp.MessageProcessor
 }

 // doReadMetrics reads metrics from all configure IPMI hosts.
@@ -213,7 +217,7 @@ func (r *IPMIReceiver) doReadMetric() {
 					continue
 				}

-				y, err := lp.New(
+				y, err := lp.NewMessage(
 					metric,
 					map[string]string{
 						"hostname": host,
@@ -230,7 +234,14 @@ func (r *IPMIReceiver) doReadMetric() {
 					},
 					time.Now())
 				if err == nil {
-					r.sink <- y
+					mc, err := clientConfig.mp.ProcessMessage(y)
+					if err == nil && mc != nil {
+						m, err := r.mp.ProcessMessage(mc)
+						if err == nil && m != nil {
+							r.sink <- m
+						}
+					}
+
 				}
 			}

@@ -296,11 +307,12 @@ func (r *IPMIReceiver) Close() {
 // NewIPMIReceiver creates a new instance of the redfish receiver
 // Initialize the receiver by giving it a name and reading in the config JSON
 func NewIPMIReceiver(name string, config json.RawMessage) (Receiver, error) {
+	var err error
 	r := new(IPMIReceiver)

 	// Config options from config file
 	configJSON := struct {
-		Type string `json:"type"`
+		defaultReceiverConfig

 		// How often the IPMI sensor metrics should be read and send to the sink (default: 30 s)
 		IntervalString string `json:"interval,omitempty"`
@@ -331,7 +343,8 @@ func NewIPMIReceiver(name string, config json.RawMessage) (Receiver, error) {
 			ExcludeMetrics []string `json:"exclude_metrics,omitempty"`

 			// Additional command line options for ipmi-sensors
-			CLIOptions []string `json:"cli_options,omitempty"`
+			CLIOptions       []string        `json:"cli_options,omitempty"`
+			MessageProcessor json.RawMessage `json:"process_messages,omitempty"`
 		} `json:"client_config"`
 	}{
 		// Set defaults values
@@ -347,8 +360,15 @@ func NewIPMIReceiver(name string, config json.RawMessage) (Receiver, error) {
 	// Create done channel
 	r.done = make(chan bool)

+	p, err := mp.NewMessageProcessor()
+	if err != nil {
+		return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
+	}
+	r.mp = p
+
 	// Set static information
-	r.meta = map[string]string{"source": r.name}
+	//r.meta = map[string]string{"source": r.name}
+	r.mp.AddAddMetaByCondition("true", "source", r.name)

 	// Read the IPMI receiver specific JSON config
 	if len(config) > 0 {
@@ -360,12 +380,18 @@ func NewIPMIReceiver(name string, config json.RawMessage) (Receiver, error) {
 		}
 	}

+	if len(r.config.MessageProcessor) > 0 {
+		err = r.mp.FromConfigJSON(r.config.MessageProcessor)
+		if err != nil {
+			return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+		}
+	}
 	// Convert interval string representation to duration
-	var err error
+
 	r.config.Interval, err = time.ParseDuration(configJSON.IntervalString)
 	if err != nil {
 		err := fmt.Errorf(
-			"Failed to parse duration string interval='%s': %w",
+			"failed to parse duration string interval='%s': %w",
 			configJSON.IntervalString,
 			err,
 		)
@@ -506,6 +532,16 @@ func NewIPMIReceiver(name string, config json.RawMessage) (Receiver, error) {
 		for _, key := range configJSON.ExcludeMetrics {
 			isExcluded[key] = true
 		}
+		p, err := mp.NewMessageProcessor()
+		if err != nil {
+			return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
+		}
+		if len(clientConfigJSON.MessageProcessor) > 0 {
+			err = p.FromConfigJSON(clientConfigJSON.MessageProcessor)
+			if err != nil {
+				return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+			}
+		}

 		r.config.ClientConfigs = append(
 			r.config.ClientConfigs,
@@ -520,6 +556,7 @@ func NewIPMIReceiver(name string, config json.RawMessage) (Receiver, error) {
 				Password:         password,
 				CLIOptions:       cliOptions,
 				isExcluded:       isExcluded,
+				mp:               p,
 			})
 	}

--- a/receivers/metricReceiver.go
+++ b/receivers/metricReceiver.go
@@ -1,11 +1,15 @@
 package receivers

 import (
-	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
+	"encoding/json"
+
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
 )

 type defaultReceiverConfig struct {
-	Type string `json:"type"`
+	Type             string          `json:"type"`
+	MessageProcessor json.RawMessage `json:"process_messages,omitempty"`
 }

 // Receiver configuration: Listen address, port
@@ -19,14 +23,15 @@ type ReceiverConfig struct {

 type receiver struct {
 	name string
-	sink chan lp.CCMetric
+	sink chan lp.CCMessage
+	mp   mp.MessageProcessor
 }

 type Receiver interface {
 	Start()
-	Close()                        // Close / finish metric receiver
-	Name() string                  // Name of the metric receiver
-	SetSink(sink chan lp.CCMetric) // Set sink channel
+	Close()                         // Close / finish metric receiver
+	Name() string                   // Name of the metric receiver
+	SetSink(sink chan lp.CCMessage) // Set sink channel
 }

 // Name returns the name of the metric receiver
@@ -35,6 +40,6 @@ func (r *receiver) Name() string {
 }

 // SetSink set the sink channel
-func (r *receiver) SetSink(sink chan lp.CCMetric) {
+func (r *receiver) SetSink(sink chan lp.CCMessage) {
 	r.sink = sink
 }
--- a/receivers/natsReceiver.go
+++ b/receivers/natsReceiver.go
@@ -4,25 +4,30 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"time"

+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
 	influx "github.com/influxdata/line-protocol/v2/lineprotocol"
 	nats "github.com/nats-io/nats.go"
 )

 type NatsReceiverConfig struct {
-	Type    string `json:"type"`
+	defaultReceiverConfig
 	Addr    string `json:"address"`
 	Port    string `json:"port"`
 	Subject string `json:"subject"`
+	User     string `json:"user,omitempty"`
+	Password string `json:"password,omitempty"`
+	NkeyFile string `json:"nkey_file,omitempty"`
 }

 type NatsReceiver struct {
 	receiver
-	nc     *nats.Conn
-	meta   map[string]string
+	nc *nats.Conn
+	//meta   map[string]string
 	config NatsReceiverConfig
 }

@@ -36,65 +41,68 @@ func (r *NatsReceiver) Start() {
 // _NatsReceive receives subscribed messages from the NATS server
 func (r *NatsReceiver) _NatsReceive(m *nats.Msg) {

-	d := influx.NewDecoderWithBytes(m.Data)
-	for d.Next() {
+	if r.sink != nil {
+		d := influx.NewDecoderWithBytes(m.Data)
+		for d.Next() {

-		// Decode measurement name
-		measurement, err := d.Measurement()
-		if err != nil {
-			msg := "_NatsReceive: Failed to decode measurement: " + err.Error()
-			cclog.ComponentError(r.name, msg)
-			return
-		}
-
-		// Decode tags
-		tags := make(map[string]string)
-		for {
-			key, value, err := d.NextTag()
+			// Decode measurement name
+			measurement, err := d.Measurement()
 			if err != nil {
-				msg := "_NatsReceive: Failed to decode tag: " + err.Error()
+				msg := "_NatsReceive: Failed to decode measurement: " + err.Error()
 				cclog.ComponentError(r.name, msg)
 				return
 			}
-			if key == nil {
-				break
-			}
-			tags[string(key)] = string(value)
-		}

-		// Decode fields
-		fields := make(map[string]interface{})
-		for {
-			key, value, err := d.NextField()
+			// Decode tags
+			tags := make(map[string]string)
+			for {
+				key, value, err := d.NextTag()
+				if err != nil {
+					msg := "_NatsReceive: Failed to decode tag: " + err.Error()
+					cclog.ComponentError(r.name, msg)
+					return
+				}
+				if key == nil {
+					break
+				}
+				tags[string(key)] = string(value)
+			}
+
+			// Decode fields
+			fields := make(map[string]interface{})
+			for {
+				key, value, err := d.NextField()
+				if err != nil {
+					msg := "_NatsReceive: Failed to decode field: " + err.Error()
+					cclog.ComponentError(r.name, msg)
+					return
+				}
+				if key == nil {
+					break
+				}
+				fields[string(key)] = value.Interface()
+			}
+
+			// Decode time stamp
+			t, err := d.Time(influx.Nanosecond, time.Time{})
 			if err != nil {
-				msg := "_NatsReceive: Failed to decode field: " + err.Error()
+				msg := "_NatsReceive: Failed to decode time: " + err.Error()
 				cclog.ComponentError(r.name, msg)
 				return
 			}
-			if key == nil {
-				break
+
+			y, _ := lp.NewMessage(
+				string(measurement),
+				tags,
+				nil,
+				fields,
+				t,
+			)
+
+			m, err := r.mp.ProcessMessage(y)
+			if err == nil && m != nil {
+				r.sink <- m
 			}
-			fields[string(key)] = value.Interface()
-		}
-
-		// Decode time stamp
-		t, err := d.Time(influx.Nanosecond, time.Time{})
-		if err != nil {
-			msg := "_NatsReceive: Failed to decode time: " + err.Error()
-			cclog.ComponentError(r.name, msg)
-			return
-		}
-
-		y, _ := lp.New(
-			string(measurement),
-			tags,
-			r.meta,
-			fields,
-			t,
-		)
-
-		if r.sink != nil {
-			r.sink <- y
 		}
 	}
 }
@@ -109,6 +117,7 @@ func (r *NatsReceiver) Close() {

 // NewNatsReceiver creates a new Receiver which subscribes to messages from a NATS server
 func NewNatsReceiver(name string, config json.RawMessage) (Receiver, error) {
+	var uinfo nats.Option = nil
 	r := new(NatsReceiver)
 	r.name = fmt.Sprintf("NatsReceiver(%s)", name)

@@ -127,16 +136,40 @@ func NewNatsReceiver(name string, config json.RawMessage) (Receiver, error) {
 		len(r.config.Subject) == 0 {
 		return nil, errors.New("not all configuration variables set required by NatsReceiver")
 	}
+	p, err := mp.NewMessageProcessor()
+	if err != nil {
+		return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
+	}
+	r.mp = p
+	if len(r.config.MessageProcessor) > 0 {
+		err = r.mp.FromConfigJSON(r.config.MessageProcessor)
+		if err != nil {
+			return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+		}
+	}

 	// Set metadata
-	r.meta = map[string]string{
-		"source": r.name,
+	// r.meta = map[string]string{
+	// 	"source": r.name,
+	// }
+	r.mp.AddAddMetaByCondition("true", "source", r.name)
+
+	if len(r.config.User) > 0 && len(r.config.Password) > 0 {
+		uinfo = nats.UserInfo(r.config.User, r.config.Password)
+	} else if len(r.config.NkeyFile) > 0 {
+		_, err := os.Stat(r.config.NkeyFile)
+		if err == nil {
+			uinfo = nats.UserCredentials(r.config.NkeyFile)
+		} else {
+			cclog.ComponentError(r.name, "NKEY file", r.config.NkeyFile, "does not exist: %v", err.Error())
+			return nil, err
+		}
 	}

 	// Connect to NATS server
 	url := fmt.Sprintf("nats://%s:%s", r.config.Addr, r.config.Port)
 	cclog.ComponentDebug(r.name, "NewNatsReceiver", url, "Subject", r.config.Subject)
-	if nc, err := nats.Connect(url); err == nil {
+	if nc, err := nats.Connect(url, uinfo); err == nil {
 		r.nc = nc
 	} else {
 		r.nc = nil
--- a/receivers/natsReceiver.md
+++ b/receivers/natsReceiver.md
@@ -10,7 +10,10 @@ The `nats` receiver can be used receive metrics from the NATS network. The `nats
    "type": "nats",
    "address" : "nats-server.example.org",
    "port" : "4222",
-    "subject" : "subject"
+    "subject" : "subject",
+    "user": "natsuser",
+    "password": "natssecret",
+    "nkey_file": "/path/to/nkey_file"
  }
 }
 ```
@@ -19,6 +22,9 @@ The `nats` receiver can be used receive metrics from the NATS network. The `nats
 - `address`: Address of the NATS control server
 - `port`: Port of the NATS control server
 - `subject`: Subscribes to this subject and receive metrics
+- `user`: Connect to nats using this user
+- `password`: Connect to nats using this password
+- `nkey_file`: Path to credentials file with NKEY

 ### Debugging

--- a/receivers/prometheusReceiver.go
+++ b/receivers/prometheusReceiver.go
@@ -13,7 +13,7 @@ import (
 	"time"

 	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )

 type PrometheusReceiverConfig struct {
@@ -74,7 +74,7 @@ func (r *PrometheusReceiver) Start() {
 					}
 					value, err := strconv.ParseFloat(lineSplit[1], 64)
 					if err == nil {
-						y, err := lp.New(name, tags, r.meta, map[string]interface{}{"value": value}, t)
+						y, err := lp.NewMessage(name, tags, r.meta, map[string]interface{}{"value": value}, t)
 						if err == nil {
 							r.sink <- y
 						}
--- a/receivers/receiveManager.go
+++ b/receivers/receiveManager.go
@@ -7,7 +7,7 @@ import (
 	"sync"

 	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )

 var AvailableReceivers = map[string]func(name string, config json.RawMessage) (Receiver, error){
@@ -19,14 +19,14 @@ var AvailableReceivers = map[string]func(name string, config json.RawMessage) (R

 type receiveManager struct {
 	inputs []Receiver
-	output chan lp.CCMetric
+	output chan lp.CCMessage
 	config []json.RawMessage
 }

 type ReceiveManager interface {
 	Init(wg *sync.WaitGroup, receiverConfigFile string) error
 	AddInput(name string, rawConfig json.RawMessage) error
-	AddOutput(output chan lp.CCMetric)
+	AddOutput(output chan lp.CCMessage)
 	Start()
 	Close()
 }
@@ -93,7 +93,7 @@ func (rm *receiveManager) AddInput(name string, rawConfig json.RawMessage) error
 	return nil
 }

-func (rm *receiveManager) AddOutput(output chan lp.CCMetric) {
+func (rm *receiveManager) AddOutput(output chan lp.CCMessage) {
 	rm.output = output
 	for _, r := range rm.inputs {
 		r.SetSink(rm.output)
--- a/receivers/redfishReceiver.go
+++ b/receivers/redfishReceiver.go
@@ -13,9 +13,10 @@ import (
 	"sync"
 	"time"

+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 	"github.com/ClusterCockpit/cc-metric-collector/pkg/hostlist"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"

 	// See: https://pkg.go.dev/github.com/stmcginnis/gofish
 	"github.com/stmcginnis/gofish"
@@ -42,6 +43,8 @@ type RedfishReceiverClientConfig struct {
 	readSensorURLs map[string][]string

 	gofish gofish.ClientConfig
+
+	mp mp.MessageProcessor
 }

 // RedfishReceiver configuration:
@@ -49,6 +52,7 @@ type RedfishReceiver struct {
 	receiver

 	config struct {
+		defaultReceiverConfig
 		fanout      int
 		Interval    time.Duration
 		HttpTimeout time.Duration
@@ -79,13 +83,19 @@ func setMetricValue(value any) map[string]interface{} {
 }

 // sendMetric sends the metric through the sink channel
-func (r *RedfishReceiver) sendMetric(name string, tags map[string]string, meta map[string]string, value any, timestamp time.Time) {
+func (r *RedfishReceiver) sendMetric(mp mp.MessageProcessor, name string, tags map[string]string, meta map[string]string, value any, timestamp time.Time) {

 	deleteEmptyTags(tags)
 	deleteEmptyTags(meta)
-	y, err := lp.New(name, tags, meta, setMetricValue(value), timestamp)
+	y, err := lp.NewMessage(name, tags, meta, setMetricValue(value), timestamp)
 	if err == nil {
-		r.sink <- y
+		mc, err := mp.ProcessMessage(y)
+		if err == nil && mc != nil {
+			m, err := r.mp.ProcessMessage(mc)
+			if err == nil && m != nil {
+				r.sink <- m
+			}
+		}
 	}
 }

@@ -119,7 +129,7 @@ func (r *RedfishReceiver) readSensors(
 			"unit":   "degC",
 		}

-		r.sendMetric("temperature", tags, meta, sensor.Reading, time.Now())
+		r.sendMetric(clientConfig.mp, "temperature", tags, meta, sensor.Reading, time.Now())
 	}

 	writeFanSpeedSensor := func(sensor *redfish.Sensor) {
@@ -145,7 +155,7 @@ func (r *RedfishReceiver) readSensors(
 			"unit":   string(sensor.ReadingUnits),
 		}

-		r.sendMetric("fan_speed", tags, meta, sensor.Reading, time.Now())
+		r.sendMetric(clientConfig.mp, "fan_speed", tags, meta, sensor.Reading, time.Now())
 	}

 	writePowerSensor := func(sensor *redfish.Sensor) {
@@ -172,7 +182,7 @@ func (r *RedfishReceiver) readSensors(
 			"unit":   "watts",
 		}

-		r.sendMetric("power", tags, meta, sensor.Reading, time.Now())
+		r.sendMetric(clientConfig.mp, "power", tags, meta, sensor.Reading, time.Now())
 	}

 	if _, ok := clientConfig.readSensorURLs[chassis.ID]; !ok {
@@ -340,7 +350,7 @@ func (r *RedfishReceiver) readThermalMetrics(
 		// ReadingCelsius shall be the current value of the temperature sensor's reading.
 		value := temperature.ReadingCelsius

-		r.sendMetric("temperature", tags, meta, value, timestamp)
+		r.sendMetric(clientConfig.mp, "temperature", tags, meta, value, timestamp)
 	}

 	for _, fan := range thermal.Fans {
@@ -381,7 +391,7 @@ func (r *RedfishReceiver) readThermalMetrics(
 			"unit":   string(fan.ReadingUnits),
 		}

-		r.sendMetric("fan_speed", tags, meta, fan.Reading, timestamp)
+		r.sendMetric(clientConfig.mp, "fan_speed", tags, meta, fan.Reading, timestamp)
 	}

 	return nil
@@ -479,7 +489,7 @@ func (r *RedfishReceiver) readPowerMetrics(
 		}

 		for name, value := range metrics {
-			r.sendMetric(name, tags, meta, value, timestamp)
+			r.sendMetric(clientConfig.mp, name, tags, meta, value, timestamp)
 		}
 	}

@@ -561,7 +571,7 @@ func (r *RedfishReceiver) readProcessorMetrics(
 	if !clientConfig.isExcluded[namePower] &&
 		// Some servers return "ConsumedPowerWatt":65535 instead of "ConsumedPowerWatt":null
 		processorMetrics.ConsumedPowerWatt != 65535 {
-		r.sendMetric(namePower, tags, metaPower, processorMetrics.ConsumedPowerWatt, timestamp)
+		r.sendMetric(clientConfig.mp, namePower, tags, metaPower, processorMetrics.ConsumedPowerWatt, timestamp)
 	}
 	// Set meta data tags
 	metaThermal := map[string]string{
@@ -573,7 +583,7 @@ func (r *RedfishReceiver) readProcessorMetrics(
 	nameThermal := "temperature"

 	if !clientConfig.isExcluded[nameThermal] {
-		r.sendMetric(nameThermal, tags, metaThermal, processorMetrics.TemperatureCelsius, timestamp)
+		r.sendMetric(clientConfig.mp, nameThermal, tags, metaThermal, processorMetrics.TemperatureCelsius, timestamp)
 	}
 	return nil
 }
@@ -776,11 +786,13 @@ func (r *RedfishReceiver) Close() {
 // NewRedfishReceiver creates a new instance of the redfish receiver
 // Initialize the receiver by giving it a name and reading in the config JSON
 func NewRedfishReceiver(name string, config json.RawMessage) (Receiver, error) {
+	var err error
 	r := new(RedfishReceiver)

 	// Config options from config file
 	configJSON := struct {
-		Type string `json:"type"`
+		Type             string          `json:"type"`
+		MessageProcessor json.RawMessage `json:"process_messages,omitempty"`

 		// Maximum number of simultaneous redfish connections (default: 64)
 		Fanout int `json:"fanout,omitempty"`
@@ -820,7 +832,8 @@ func NewRedfishReceiver(name string, config json.RawMessage) (Receiver, error) {
 			DisableThermalMetrics   bool `json:"disable_thermal_metrics"`

 			// Per client excluded metrics
-			ExcludeMetrics []string `json:"exclude_metrics,omitempty"`
+			ExcludeMetrics   []string        `json:"exclude_metrics,omitempty"`
+			MessageProcessor json.RawMessage `json:"process_messages,omitempty"`
 		} `json:"client_config"`
 	}{
 		// Set defaults values
@@ -846,13 +859,24 @@ func NewRedfishReceiver(name string, config json.RawMessage) (Receiver, error) {
 			return nil, err
 		}
 	}
+	p, err := mp.NewMessageProcessor()
+	if err != nil {
+		return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
+	}
+	r.mp = p
+	if len(r.config.MessageProcessor) > 0 {
+		err = r.mp.FromConfigJSON(r.config.MessageProcessor)
+		if err != nil {
+			return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+		}
+	}

 	// Convert interval string representation to duration
-	var err error
+
 	r.config.Interval, err = time.ParseDuration(configJSON.IntervalString)
 	if err != nil {
 		err := fmt.Errorf(
-			"Failed to parse duration string interval='%s': %w",
+			"failed to parse duration string interval='%s': %w",
 			configJSON.IntervalString,
 			err,
 		)
@@ -864,7 +888,7 @@ func NewRedfishReceiver(name string, config json.RawMessage) (Receiver, error) {
 	r.config.HttpTimeout, err = time.ParseDuration(configJSON.HttpTimeoutString)
 	if err != nil {
 		err := fmt.Errorf(
-			"Failed to parse duration string http_timeout='%s': %w",
+			"failed to parse duration string http_timeout='%s': %w",
 			configJSON.HttpTimeoutString,
 			err,
 		)
@@ -948,6 +972,18 @@ func NewRedfishReceiver(name string, config json.RawMessage) (Receiver, error) {
 		for _, key := range configJSON.ExcludeMetrics {
 			isExcluded[key] = true
 		}
+		p, err = mp.NewMessageProcessor()
+		if err != nil {
+			cclog.ComponentError(r.name, err.Error())
+			return nil, err
+		}
+		if len(clientConfigJSON.MessageProcessor) > 0 {
+			err = p.FromConfigJSON(clientConfigJSON.MessageProcessor)
+			if err != nil {
+				cclog.ComponentError(r.name, err.Error())
+				return nil, err
+			}
+		}

 		hostList, err := hostlist.Expand(clientConfigJSON.HostList)
 		if err != nil {
@@ -978,6 +1014,7 @@ func NewRedfishReceiver(name string, config json.RawMessage) (Receiver, error) {
 						Endpoint:   endpoint,
 						HTTPClient: httpClient,
 					},
+					mp: p,
 				})
 		}

@@ -1002,7 +1039,7 @@ func NewRedfishReceiver(name string, config json.RawMessage) (Receiver, error) {
 	for i := range r.config.ClientConfigs {
 		host := r.config.ClientConfigs[i].Hostname
 		if isDuplicate[host] {
-			err := fmt.Errorf("Found duplicate client config for host %s", host)
+			err := fmt.Errorf("found duplicate client config for host %s", host)
 			cclog.ComponentError(r.name, err)
 			return nil, err
 		}
--- a/receivers/sampleReceiver.go
+++ b/receivers/sampleReceiver.go
@@ -5,11 +5,13 @@ import (
 	"fmt"

 	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
 )

 // SampleReceiver configuration: receiver type, listen address, port
+// The defaultReceiverConfig contains the keys 'type' and 'process_messages'
 type SampleReceiverConfig struct {
-	Type string `json:"type"`
+	defaultReceiverConfig
 	Addr string `json:"address"`
 	Port string `json:"port"`
 }
@@ -19,7 +21,6 @@ type SampleReceiver struct {
 	config SampleReceiverConfig

 	// Storage for static information
-	meta map[string]string
 	// Use in case of own go routine
 	// done chan bool
 	// wg   sync.WaitGroup
@@ -79,8 +80,19 @@ func NewSampleReceiver(name string, config json.RawMessage) (Receiver, error) {
 	// The name should be chosen in such a way that different instances of SampleReceiver can be distinguished
 	r.name = fmt.Sprintf("SampleReceiver(%s)", name)

+	// create new message processor
+	p, err := mp.NewMessageProcessor()
+	if err != nil {
+		cclog.ComponentError(r.name, "Initialization of message processor failed:", err.Error())
+		return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
+	}
+	r.mp = p
 	// Set static information
-	r.meta = map[string]string{"source": r.name}
+	err = r.mp.AddAddMetaByCondition("true", "source", r.name)
+	if err != nil {
+		cclog.ComponentError(r.name, fmt.Sprintf("Failed to add static information source=%s:", r.name), err.Error())
+		return nil, fmt.Errorf("failed to add static information source=%s: %v", r.name, err.Error())
+	}

 	// Set defaults in r.config
 	// Allow overwriting these defaults by reading config JSON
@@ -94,6 +106,15 @@ func NewSampleReceiver(name string, config json.RawMessage) (Receiver, error) {
 		}
 	}

+	// Add message processor config
+	if len(r.config.MessageProcessor) > 0 {
+		err = r.mp.FromConfigJSON(r.config.MessageProcessor)
+		if err != nil {
+			cclog.ComponentError(r.name, "Failed parsing JSON for message processor:", err.Error())
+			return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+		}
+	}
+
 	// Check that all required fields in the configuration are set
 	// Use 'if len(r.config.Option) > 0' for strings