Merge develop branch into main (#123)

* Add cpu_used (all-cpu_idle) to CpustatCollector * Update cc-metric-collector.init * Allow selection of timestamp precision in HttpSink * Add comment about precision requirement for cc-metric-store * Fix for API changes in gofish@v0.15.0 * Update requirements to latest version * Read sensors through redfish * Update golang toolchain to 1.21 * Remove stray error check * Update main config in configuration.md * Update Release action to use golang 1.22 stable release, no golang RPMs anymore * Update runonce action to use golang 1.22 stable release, no golang RPMs anymore * Update README.md Use right JSON type in configuration * Update sink's README * Test whether ipmitool or ipmi-sensors can be executed without errors * Little fixes to the prometheus sink (#115) * Add uint64 to float64 cast option * Add prometheus sink to the list of available sinks * Add aggregated counters by gpu for nvlink errors --------- Co-authored-by: Michael Schwarz <schwarz@uni-paderborn.de> * Ccmessage migration (#119) * Add cpu_used (all-cpu_idle) to CpustatCollector * Update cc-metric-collector.init * Allow selection of timestamp precision in HttpSink * Add comment about precision requirement for cc-metric-store * Fix for API changes in gofish@v0.15.0 * Update requirements to latest version * Read sensors through redfish * Update golang toolchain to 1.21 * Remove stray error check * Update main config in configuration.md * Update Release action to use golang 1.22 stable release, no golang RPMs anymore * Update runonce action to use golang 1.22 stable release, no golang RPMs anymore * Switch to CCMessage for all files. --------- Co-authored-by: Holger Obermaier <Holger.Obermaier@kit.edu> Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> * Switch to ccmessage also for latest additions in nvidiaMetric * New Message processor (#118) * Add cpu_used (all-cpu_idle) to CpustatCollector * Update cc-metric-collector.init * Allow selection of timestamp precision in HttpSink * Add comment about precision requirement for cc-metric-store * Fix for API changes in gofish@v0.15.0 * Update requirements to latest version * Read sensors through redfish * Update golang toolchain to 1.21 * Remove stray error check * Update main config in configuration.md * Update Release action to use golang 1.22 stable release, no golang RPMs anymore * Update runonce action to use golang 1.22 stable release, no golang RPMs anymore * New message processor to check whether a message should be dropped or manipulate it in flight * Create a copy of message before manipulation --------- Co-authored-by: Holger Obermaier <Holger.Obermaier@kit.edu> Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> * Update collector's Makefile and go.mod/sum files * Use message processor in router, all sinks and all receivers * Add support for credential file (NKEY) to NATS sink and receiver * Fix JSON keys in message processor configuration * Update docs for message processor, router and the default router config file * Add link to expr syntax and fix regex matching docs * Update sample collectors * Minor style change in collector manager * Some helpers for ccTopology * LIKWID collector: write log owner change only once * Fix for metrics without units and reduce debugging messages for messageProcessor * Use shorted hostname for hostname added by router * Define default port for NATS * CPUstat collector: only add unit for applicable metrics * Add precision option to all sinks using Influx's encoder * Add message processor to all sink documentation * Add units to documentation of cpustat collector --------- Co-authored-by: Holger Obermaier <Holger.Obermaier@kit.edu> Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com> Co-authored-by: oscarminus <me@oscarminus.de> Co-authored-by: Michael Schwarz <schwarz@uni-paderborn.de>
2025-07-23 05:11:40 +02:00 · 2024-12-19 23:00:14 +01:00
parent 21646e1edf
commit 7840de7b82
74 changed files with 1902 additions and 1017 deletions
--- a/pkg/ccTopology/ccTopology.go
+++ b/pkg/ccTopology/ccTopology.go
@@ -296,6 +296,25 @@ func GetTypeList(topology_type string) []int {
 	return []int{}
 }

+func GetTypeId(hwt HwthreadEntry, topology_type string) (int, error) {
+	var err error = nil
+	switch topology_type {
+	case "node":
+		return 0, err
+	case "socket":
+		return hwt.Socket, err
+	case "die":
+		return hwt.Die, err
+	case "memoryDomain":
+		return hwt.NumaDomain, err
+	case "core":
+		return hwt.Core, err
+	case "hwthread":
+		return hwt.CpuID, err
+	}
+	return -1, fmt.Errorf("unknown topology type '%s'", topology_type)
+}
+
 // CpuData returns CPU data for each hardware thread
 func CpuData() []HwthreadEntry {
 	// return a deep copy to protect cache data
@@ -423,3 +442,22 @@ func GetCoreHwthreads(core int) []int {
 	}
 	return cpuList
 }
+
+// GetTypeList gets the list of specified type using the naming format inside ClusterCockpit
+func GetTypeHwthreads(topology_type string, id int) []int {
+	switch topology_type {
+	case "node":
+		return HwthreadList()
+	case "socket":
+		return GetSocketHwthreads(id)
+	case "die":
+		return GetDieHwthreads(id)
+	case "memoryDomain":
+		return GetNumaDomainHwthreads(id)
+	case "core":
+		return GetCoreHwthreads(id)
+	case "hwthread":
+		return []int{id}
+	}
+	return []int{}
+}
--- a/pkg/messageProcessor/README.md
+++ b/pkg/messageProcessor/README.md
@@ -10,6 +10,8 @@ lack of flexibility caused some trouble:
 > resolution for some metrics. The issue was basically the `mem_used` metric showing the currently used memory of the node. Ganglia wants it in `kByte` as provided
 > by the Linux operating system but CC wanted it in `GByte`.

+With the message processor, the Ganglia sinks can apply the unit prefix changes individually and name the metrics as required by Ganglia.
+
 ## For developers

 Whenever you receive or are about to send a message out, you should provide some processing.
@@ -20,41 +22,211 @@ New operations can be added to the message processor at runtime. Of course, they
 or some fields in a configuration file for the processing.

 The message processor uses the following configuration
-```golang
-type messageProcessorConfig struct {
-	DropMessages     []string          `json:"drop_messages"`      // List of metric names to drop. For fine-grained dropping use drop_messages_if
-	DropMessagesIf   []string          `json:"drop_messages_if"`   // List of evaluatable terms to drop messages
-	RenameMessages   map[string]string `json:"rename_messages"`    // Map to rename metric name from key to value
-	NormalizeUnits   bool              `json:"normalize_units"`    // Check unit meta flag and normalize it using cc-units
-	ChangeUnitPrefix map[string]string `json:"change_unit_prefix"` // Add prefix that should be applied to the messages
+
+```json
+{
+	"drop_messages": [
+		"name_of_message_to_drop"
+	],
+	"drop_messages_if": [
+		"condition_when_to_drop_message",
+		"name == 'drop_this'",
+		"tag.hostname == 'this_host'",
+		"meta.unit != 'MB'"
+	],
+	"rename_messages" : {
+		"old_message_name" : "new_message_name"
+	},
+	"rename_messages_if": {
+		"condition_when_to_rename_message" : "new_name"
+	},
+	"add_tags_if": [
+		{
+			"if" : "condition_when_to_add_tag",
+			"key": "name_for_new_tag",
+			"value": "new_tag_value"
+		}
+	],
+	"delete_tags_if": [
+		{
+			"if" : "condition_when_to_delete_tag",
+			"key": "name_of_tag"
+		}
+	],
+	"add_meta_if": [
+		{
+			"if" : "condition_when_to_add_meta_info",
+			"key": "name_for_new_meta_info",
+			"value": "new_meta_info_value"
+		}
+	],
+	"delete_meta_if": [
+		{
+			"if" : "condition_when_to_delete_meta_info",
+			"key": "name_of_meta_info"
+		}
+	],
+	"add_field_if": [
+		{
+			"if" : "condition_when_to_add_field",
+			"key": "name_for_new_field",
+			"value": "new_field_value_but_only_string_at_the_moment"
+		}
+	],
+	"delete_field_if": [
+		{
+			"if" : "condition_when_to_delete_field",
+			"key": "name_of_field"
+		}
+	],
+	"move_tag_to_meta_if": [
+		{
+			"if" : "condition_when_to_move_tag_to_meta_info_including_its_value",
+			"key": "name_of_tag",
+			"value": "name_of_meta_info"
+		}
+	],
+	"move_tag_to_field_if": [
+		{
+			"if" : "condition_when_to_move_tag_to_fields_including_its_value",
+			"key": "name_of_tag",
+			"value": "name_of_field"
+		}
+	],
+	"move_meta_to_tag_if": [
+		{
+			"if" : "condition_when_to_move_meta_info_to_tags_including_its_value",
+			"key": "name_of_meta_info",
+			"value": "name_of_tag"
+		}
+	],
+	"move_meta_to_field_if": [
+		{
+			"if" : "condition_when_to_move_meta_info_to_fields_including_its_value",
+			"key": "name_of_tag",
+			"value": "name_of_meta_info"
+		}
+	],
+	"move_field_to_tag_if": [
+		{
+			"if" : "condition_when_to_move_field_to_tags_including_its_stringified_value",
+			"key": "name_of_field",
+			"value": "name_of_tag"
+		}
+	],
+	"move_field_to_meta_if": [
+		{
+			"if" : "condition_when_to_move_field_to_meta_info_including_its_stringified_value",
+			"key": "name_of_field",
+			"value": "name_of_meta_info"
+		}
+	],
+	"drop_by_message_type": [
+		"metric",
+		"event",
+		"log",
+		"control"
+	],
+	"change_unit_prefix": {
+		"name == 'metric_with_wrong_unit_prefix'" : "G",
+		"only_if_messagetype == 'metric'": "T"
+	},
+	"normalize_units": true,
+	"add_base_env": {
+		"MY_CONSTANT_FOR_CUSTOM_CONDITIONS": 1.0,
+		"output_value_for_test_metrics": 42.0,
+	},
+	"stage_order": [
+		"rename_messages_if",
+		"drop_messages"
+	]
 }
 ```

+The options `change_unit_prefix` and `normalize_units` are only applied to CCMetrics. It is not possible to delete the field related to each message type as defined in [cc-specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/interfaces/lineprotocol). In short:
+- CCMetrics always have to have a field named `value`
+- CCEvents always have to have a field named `event`
+- CCLogs always have to have a field named `log`
+- CCControl messages always have to have a field named `control`
+
+With `add_base_env`, one can specifiy mykey=myvalue pairs that can be used in conditions like `tag.type == mykey`.
+
+The order in which each message is processed, can be specified with the `stage_order` option. The stage names are the keys in the JSON configuration, thus `change_unit_prefix`, `move_field_to_meta_if`, etc. Stages can be listed multiple times.
+
+### Using the component
 In order to load the configuration from a `json.RawMessage`:
 ```golang
-mp, _ := NewMessageProcessor()
-
+mp, err := NewMessageProcessor()
+if err != nil {
+	log.Error("failed to create new message processor")
+}
 mp.FromConfigJSON(configJson)
 ```

-### Using the component
 After initialization and adding the different operations, the `ProcessMessage()` function applies all operations and returns whether the message should be dropped.

 ```golang
 m := lp.CCMetric{}

-drop, err := mp.ProcessMessage(m)
-if !drop {
-    // process further
+x, err := mp.ProcessMessage(m)
+if err != nil {
+	// handle error
+}
+if x != nil {
+    // process x further
+} else {
+	// this message got dropped
 }
 ```

-#### Overhead
+Single operations can be added and removed at runtime
+```golang
+type MessageProcessor interface {
+	// Functions to set the execution order of the processing stages
+	SetStages([]string) error
+	DefaultStages() []string
+	// Function to add variables to the base evaluation environment
+	AddBaseEnv(env map[string]interface{}) error
+	// Functions to add and remove rules
+	AddDropMessagesByName(name string) error
+	RemoveDropMessagesByName(name string)
+	AddDropMessagesByCondition(condition string) error
+	RemoveDropMessagesByCondition(condition string)
+	AddRenameMetricByCondition(condition string, name string) error
+	RemoveRenameMetricByCondition(condition string)
+	AddRenameMetricByName(from, to string) error
+	RemoveRenameMetricByName(from string)
+	SetNormalizeUnits(settings bool)
+	AddChangeUnitPrefix(condition string, prefix string) error
+	RemoveChangeUnitPrefix(condition string)
+	AddAddTagsByCondition(condition, key, value string) error
+	RemoveAddTagsByCondition(condition string)
+	AddDeleteTagsByCondition(condition, key, value string) error
+	RemoveDeleteTagsByCondition(condition string)
+	AddAddMetaByCondition(condition, key, value string) error
+	RemoveAddMetaByCondition(condition string)
+	AddDeleteMetaByCondition(condition, key, value string) error
+	RemoveDeleteMetaByCondition(condition string)
+	AddMoveTagToMeta(condition, key, value string) error
+	RemoveMoveTagToMeta(condition string)
+	AddMoveTagToFields(condition, key, value string) error
+	RemoveMoveTagToFields(condition string)
+	AddMoveMetaToTags(condition, key, value string) error
+	RemoveMoveMetaToTags(condition string)
+	AddMoveMetaToFields(condition, key, value string) error
+	RemoveMoveMetaToFields(condition string)
+	AddMoveFieldToTags(condition, key, value string) error
+	RemoveMoveFieldToTags(condition string)
+	AddMoveFieldToMeta(condition, key, value string) error
+	RemoveMoveFieldToMeta(condition string)
+	// Read in a JSON configuration
+	FromConfigJSON(config json.RawMessage) error
+	ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, error)
+	// Processing functions for legacy CCMetric and current CCMessage
+	ProcessMetric(m lp.CCMetric) (lp2.CCMessage, error)
+}
+```

-The operations taking conditions are pre-processed, which is commonly the time consuming part but, of course, with each added operation, the time to process a message
-increases.
-
-## For users

 ### Syntax for evaluatable terms

@@ -62,17 +234,19 @@ The message processor uses `gval` for evaluating the terms. It provides a basic

 Accessible for operations are
 - `name` of the message
- `timestamp` of the message
- `type`, `type-id` of the message (also `tag_type` and `tag_type-id`)
- `stype`, `stype-id` of the message (if message has theses tags, also `tag_stype` and `tag_stype-id`)
+- `timestamp` or `time` of the message
+- `type`, `type-id` of the message (also `tag_type`, `tag_type-id` and `tag_typeid`)
+- `stype`, `stype-id` of the message (if message has theses tags, also `tag_stype`, `tag_stype-id` and `tag_stypeid`)
 - `value` for a CCMetric message (also `field_value`)
 - `event` for a CCEvent message (also `field_event`)
 - `control` for a CCControl message (also `field_control`)
 - `log` for a CCLog message (also `field_log`)
+- `messagetype` or `msgtype`. Possible values `event`, `metric`, `log` and `control`.

-Generally, all tags are accessible with `tag_<tagkey>`, all meta information with `meta_<metakey>` and fields with `field_<fieldkey>`.
+Generally, all tags are accessible with `tag_<tagkey>`, `tags_<tagkey>` or `tags.<tagkey>`. Similarly for all fields with `field[s]?[_.]<fieldkey>`. For meta information `meta[_.]<metakey>` (there is no `metas[_.]<metakey>`).

- Comparing strings: `==`, `!=`, `match(str, regex)` (use `%` instead of `\`!)
+The [syntax of `expr`](https://expr-lang.org/docs/language-definition) is accepted with some additions:
+- Comparing strings: `==`, `!=`, `str matches regex` (use `%` instead of `\`!)
 - Combining conditions: `&&`, `||`
 - Comparing numbers: `==`, `!=`, `<`, `>`, `<=`, `>=`
 - Test lists: `<value> in <list>`
@@ -82,3 +256,11 @@ Often the operations are written in JSON files for loading them at startup. In J
 - use `''` instead of `""` for strings
 - for the regexes, use `%` instead of `\`

+
+For operations that should be applied on all messages, use the condition `true`.
+
+### Overhead
+
+The operations taking conditions are pre-processed, which is commonly the time consuming part but, of course, with each added operation, the time to process a message
+increases. Moreover, the processing creates a copy of the message.
+
--- a/pkg/messageProcessor/messageProcessor.go
+++ b/pkg/messageProcessor/messageProcessor.go
@@ -6,18 +6,19 @@ import (
 	"strings"
 	"sync"

-	lp2 "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
+	lplegacy "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
+
 	"github.com/expr-lang/expr"
 	"github.com/expr-lang/expr/vm"
 )

 // Message processor add/delete tag/meta configuration
 type messageProcessorTagConfig struct {
-	Key       string `json:"key"`   // Tag name
-	Value     string `json:"value"` // Tag value
-	Condition string `json:"if"`    // Condition for adding or removing corresponding tag
+	Key       string `json:"key"`             // Tag name
+	Value     string `json:"value,omitempty"` // Tag value
+	Condition string `json:"if"`              // Condition for adding or removing corresponding tag
 }

 type messageProcessorConfig struct {
@@ -32,8 +33,8 @@ type messageProcessorConfig struct {
 	DelTagsIf        []messageProcessorTagConfig `json:"delete_tags_if"`               // List of tags that are removed when the condition is met
 	AddMetaIf        []messageProcessorTagConfig `json:"add_meta_if"`                  // List of meta infos that are added when the condition is met
 	DelMetaIf        []messageProcessorTagConfig `json:"delete_meta_if"`               // List of meta infos that are removed when the condition is met
-	AddFieldIf       []messageProcessorTagConfig `json:"add_fields_if"`                // List of fields that are added when the condition is met
-	DelFieldIf       []messageProcessorTagConfig `json:"delete_fields_if"`             // List of fields that are removed when the condition is met
+	AddFieldIf       []messageProcessorTagConfig `json:"add_field_if"`                 // List of fields that are added when the condition is met
+	DelFieldIf       []messageProcessorTagConfig `json:"delete_field_if"`              // List of fields that are removed when the condition is met
 	DropByType       []string                    `json:"drop_by_message_type"`         // List of message types that should be dropped
 	MoveTagToMeta    []messageProcessorTagConfig `json:"move_tag_to_meta_if"`
 	MoveTagToField   []messageProcessorTagConfig `json:"move_tag_to_field_if"`
@@ -117,8 +118,8 @@ type MessageProcessor interface {
 	// Read in a JSON configuration
 	FromConfigJSON(config json.RawMessage) error
 	// Processing functions for legacy CCMetric and current CCMessage
-	ProcessMetric(m lp.CCMetric) (lp2.CCMessage, error)
-	ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, error)
+	ProcessMetric(m lplegacy.CCMetric) (lp.CCMessage, error)
+	ProcessMessage(m lp.CCMessage) (lp.CCMessage, error)
 	//EvalToBool(condition string, parameters map[string]interface{}) (bool, error)
 	//EvalToFloat64(condition string, parameters map[string]interface{}) (float64, error)
 	//EvalToString(condition string, parameters map[string]interface{}) (string, error)
@@ -261,8 +262,8 @@ var baseenv = map[string]interface{}{
 		"log":     "",
 	},
 	"timestamp": 1234567890,
-	"msg":       lp2.EmptyMessage(),
-	"message":   lp2.EmptyMessage(),
+	"msg":       lp.EmptyMessage(),
+	"message":   lp.EmptyMessage(),
 }

 func addBaseEnvWalker(values map[string]interface{}) map[string]interface{} {
@@ -759,8 +760,8 @@ func (mp *messageProcessor) FromConfigJSON(config json.RawMessage) error {
 	return nil
 }

-func (mp *messageProcessor) ProcessMetric(metric lp.CCMetric) (lp2.CCMessage, error) {
-	m, err := lp2.NewMessage(
+func (mp *messageProcessor) ProcessMetric(metric lplegacy.CCMetric) (lp.CCMessage, error) {
+	m, err := lp.NewMessage(
 		metric.Name(),
 		metric.Tags(),
 		metric.Meta(),
@@ -774,9 +775,9 @@ func (mp *messageProcessor) ProcessMetric(metric lp.CCMetric) (lp2.CCMessage, er

 }

-func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, error) {
+func (mp *messageProcessor) ProcessMessage(m lp.CCMessage) (lp.CCMessage, error) {
 	var err error = nil
-	var out lp2.CCMessage = lp2.FromMessage(m)
+	var out lp.CCMessage = lp.FromMessage(m)

 	name := out.Name()

@@ -802,45 +803,45 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 		switch s {
 		case STAGENAME_DROP_BY_NAME:
 			if len(mp.dropMessages) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Dropping by message name ", name)
+				//cclog.ComponentDebug("MessageProcessor", "Dropping by message name ", name)
 				if _, ok := mp.dropMessages[name]; ok {
-					cclog.ComponentDebug("MessageProcessor", "Drop")
+					//cclog.ComponentDebug("MessageProcessor", "Drop")
 					return nil, nil
 				}
 			}
 		case STAGENAME_DROP_BY_TYPE:
 			if len(mp.dropTypes) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Dropping by message type")
+				//cclog.ComponentDebug("MessageProcessor", "Dropping by message type")
 				if _, ok := mp.dropTypes[params["messagetype"].(string)]; ok {
-					cclog.ComponentDebug("MessageProcessor", "Drop")
+					//cclog.ComponentDebug("MessageProcessor", "Drop")
 					return nil, nil
 				}
 			}
 		case STAGENAME_DROP_IF:
 			if len(mp.dropMessagesIf) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Dropping by condition")
+				//cclog.ComponentDebug("MessageProcessor", "Dropping by condition")
 				drop, err := dropMessagesIf(&params, &mp.dropMessagesIf)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
 				}
 				if drop {
-					cclog.ComponentDebug("MessageProcessor", "Drop")
+					//cclog.ComponentDebug("MessageProcessor", "Drop")
 					return nil, nil
 				}
 			}
 		case STAGENAME_RENAME_BY_NAME:
 			if len(mp.renameMessages) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Renaming by name match")
+				//cclog.ComponentDebug("MessageProcessor", "Renaming by name match")
 				if newname, ok := mp.renameMessages[name]; ok {
-					cclog.ComponentDebug("MessageProcessor", "Rename to", newname)
+					//cclog.ComponentDebug("MessageProcessor", "Rename to", newname)
 					out.SetName(newname)
-					cclog.ComponentDebug("MessageProcessor", "Add old name as 'oldname' to meta", name)
+					//cclog.ComponentDebug("MessageProcessor", "Add old name as 'oldname' to meta", name)
 					out.AddMeta("oldname", name)
 				}
 			}
 		case STAGENAME_RENAME_IF:
 			if len(mp.renameMessagesIf) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Renaming by condition")
+				//cclog.ComponentDebug("MessageProcessor", "Renaming by condition")
 				_, err := renameMessagesIf(out, &params, &mp.renameMessagesIf)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -848,7 +849,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_ADD_TAG:
 			if len(mp.addTagsIf) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Adding tags")
+				//cclog.ComponentDebug("MessageProcessor", "Adding tags")
 				_, err = addTagIf(out, &params, &mp.addTagsIf)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -856,7 +857,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_DELETE_TAG:
 			if len(mp.deleteTagsIf) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Delete tags")
+				//cclog.ComponentDebug("MessageProcessor", "Delete tags")
 				_, err = deleteTagIf(out, &params, &mp.deleteTagsIf)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -864,7 +865,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_ADD_META:
 			if len(mp.addMetaIf) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Adding meta information")
+				//cclog.ComponentDebug("MessageProcessor", "Adding meta information")
 				_, err = addMetaIf(out, &params, &mp.addMetaIf)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -872,7 +873,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_DELETE_META:
 			if len(mp.deleteMetaIf) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Delete meta information")
+				//cclog.ComponentDebug("MessageProcessor", "Delete meta information")
 				_, err = deleteMetaIf(out, &params, &mp.deleteMetaIf)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -880,7 +881,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_ADD_FIELD:
 			if len(mp.addFieldIf) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Adding fields")
+				//cclog.ComponentDebug("MessageProcessor", "Adding fields")
 				_, err = addFieldIf(out, &params, &mp.addFieldIf)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -888,7 +889,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_DELETE_FIELD:
 			if len(mp.deleteFieldIf) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Delete fields")
+				//cclog.ComponentDebug("MessageProcessor", "Delete fields")
 				_, err = deleteFieldIf(out, &params, &mp.deleteFieldIf)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -896,7 +897,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_MOVE_TAG_META:
 			if len(mp.moveTagToMeta) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Move tag to meta")
+				//cclog.ComponentDebug("MessageProcessor", "Move tag to meta")
 				_, err := moveTagToMeta(out, &params, &mp.moveTagToMeta)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -904,7 +905,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_MOVE_TAG_FIELD:
 			if len(mp.moveTagToField) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Move tag to fields")
+				//cclog.ComponentDebug("MessageProcessor", "Move tag to fields")
 				_, err := moveTagToField(out, &params, &mp.moveTagToField)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -912,7 +913,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_MOVE_META_TAG:
 			if len(mp.moveMetaToTag) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Move meta to tags")
+				//cclog.ComponentDebug("MessageProcessor", "Move meta to tags")
 				_, err := moveMetaToTag(out, &params, &mp.moveMetaToTag)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -920,7 +921,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_MOVE_META_FIELD:
 			if len(mp.moveMetaToField) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Move meta to fields")
+				//cclog.ComponentDebug("MessageProcessor", "Move meta to fields")
 				_, err := moveMetaToField(out, &params, &mp.moveMetaToField)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -928,7 +929,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_MOVE_FIELD_META:
 			if len(mp.moveFieldToMeta) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Move field to meta")
+				//cclog.ComponentDebug("MessageProcessor", "Move field to meta")
 				_, err := moveFieldToMeta(out, &params, &mp.moveFieldToMeta)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -936,7 +937,7 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_MOVE_FIELD_TAG:
 			if len(mp.moveFieldToTag) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Move field to tags")
+				//cclog.ComponentDebug("MessageProcessor", "Move field to tags")
 				_, err := moveFieldToTag(out, &params, &mp.moveFieldToTag)
 				if err != nil {
 					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -944,8 +945,8 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro
 			}
 		case STAGENAME_NORMALIZE_UNIT:
 			if mp.normalizeUnits {
-				cclog.ComponentDebug("MessageProcessor", "Normalize units")
-				if lp2.IsMetric(out) {
+				//cclog.ComponentDebug("MessageProcessor", "Normalize units")
+				if lp.IsMetric(out) {
 					_, err := normalizeUnits(out)
 					if err != nil {
 						return out, fmt.Errorf("failed to evaluate: %v", err.Error())
@@ -957,8 +958,8 @@ func (mp *messageProcessor) ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, erro

 		case STAGENAME_CHANGE_UNIT_PREFIX:
 			if len(mp.changeUnitPrefix) > 0 {
-				cclog.ComponentDebug("MessageProcessor", "Change unit prefix")
-				if lp2.IsMetric(out) {
+				//cclog.ComponentDebug("MessageProcessor", "Change unit prefix")
+				if lp.IsMetric(out) {
 					_, err := changeUnitPrefix(out, &params, &mp.changeUnitPrefix)
 					if err != nil {
 						return out, fmt.Errorf("failed to evaluate: %v", err.Error())
--- a/pkg/messageProcessor/messageProcessorFuncs.go
+++ b/pkg/messageProcessor/messageProcessorFuncs.go
@@ -5,7 +5,6 @@ import (
 	"fmt"

 	lp2 "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
-	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 	units "github.com/ClusterCockpit/cc-units"
 	"github.com/expr-lang/expr"
 	"github.com/expr-lang/expr/vm"
@@ -26,49 +25,47 @@ func moveInMessage(message lp2.CCMessage, params *map[string]interface{}, checks
 		if err != nil {
 			return false, fmt.Errorf("failed to evaluate: %v", err.Error())
 		}
-		cclog.ComponentDebug("MessageProcessor", "Move from", from, "to", to)
+		//cclog.ComponentDebug("MessageProcessor", "Move from", from, "to", to)
 		if value.(bool) {
 			var v string
 			var ok bool = false
 			switch from {
 			case MESSAGE_LOCATION_TAGS:
-				cclog.ComponentDebug("MessageProcessor", "Getting tag key", data.Key)
+				//cclog.ComponentDebug("MessageProcessor", "Getting tag key", data.Key)
 				v, ok = message.GetTag(data.Key)
 			case MESSAGE_LOCATION_META:
-				cclog.ComponentDebug("MessageProcessor", "Getting meta key", data.Key)
-				cclog.ComponentDebug("MessageProcessor", message.Meta())
+				//cclog.ComponentDebug("MessageProcessor", "Getting meta key", data.Key)
+				//cclog.ComponentDebug("MessageProcessor", message.Meta())
 				v, ok = message.GetMeta(data.Key)
 			case MESSAGE_LOCATION_FIELDS:
 				var x interface{}
-				cclog.ComponentDebug("MessageProcessor", "Getting field key", data.Key)
+				//cclog.ComponentDebug("MessageProcessor", "Getting field key", data.Key)
 				x, ok = message.GetField(data.Key)
 				v = fmt.Sprintf("%v", x)
 			}
 			if ok {
 				switch from {
 				case MESSAGE_LOCATION_TAGS:
-					cclog.ComponentDebug("MessageProcessor", "Removing tag key", data.Key)
+					//cclog.ComponentDebug("MessageProcessor", "Removing tag key", data.Key)
 					message.RemoveTag(data.Key)
 				case MESSAGE_LOCATION_META:
-					cclog.ComponentDebug("MessageProcessor", "Removing meta key", data.Key)
+					//cclog.ComponentDebug("MessageProcessor", "Removing meta key", data.Key)
 					message.RemoveMeta(data.Key)
 				case MESSAGE_LOCATION_FIELDS:
-					cclog.ComponentDebug("MessageProcessor", "Removing field key", data.Key)
+					//cclog.ComponentDebug("MessageProcessor", "Removing field key", data.Key)
 					message.RemoveField(data.Key)
 				}
 				switch to {
 				case MESSAGE_LOCATION_TAGS:
-					cclog.ComponentDebug("MessageProcessor", "Adding tag", data.Value, "->", v)
+					//cclog.ComponentDebug("MessageProcessor", "Adding tag", data.Value, "->", v)
 					message.AddTag(data.Value, v)
 				case MESSAGE_LOCATION_META:
-					cclog.ComponentDebug("MessageProcessor", "Adding meta", data.Value, "->", v)
+					//cclog.ComponentDebug("MessageProcessor", "Adding meta", data.Value, "->", v)
 					message.AddMeta(data.Value, v)
 				case MESSAGE_LOCATION_FIELDS:
-					cclog.ComponentDebug("MessageProcessor", "Adding field", data.Value, "->", v)
+					//cclog.ComponentDebug("MessageProcessor", "Adding field", data.Value, "->", v)
 					message.AddField(data.Value, v)
 				}
-			} else {
-				return false, fmt.Errorf("failed to get message entry: %s", data.Key)
 			}
 		}
 	}
@@ -88,14 +85,14 @@ func deleteIf(message lp2.CCMessage, params *map[string]interface{}, checks *map
 				case "value", "event", "log", "control":
 					return false, errors.New("cannot delete protected fields")
 				default:
-					cclog.ComponentDebug("MessageProcessor", "Removing field for", data.Key)
+					//cclog.ComponentDebug("MessageProcessor", "Removing field for", data.Key)
 					message.RemoveField(data.Key)
 				}
 			case MESSAGE_LOCATION_TAGS:
-				cclog.ComponentDebug("MessageProcessor", "Removing tag for", data.Key)
+				//cclog.ComponentDebug("MessageProcessor", "Removing tag for", data.Key)
 				message.RemoveTag(data.Key)
 			case MESSAGE_LOCATION_META:
-				cclog.ComponentDebug("MessageProcessor", "Removing meta for", data.Key)
+				//cclog.ComponentDebug("MessageProcessor", "Removing meta for", data.Key)
 				message.RemoveMeta(data.Key)
 			}
 		}
@@ -112,13 +109,13 @@ func addIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*v
 		if value.(bool) {
 			switch location {
 			case MESSAGE_LOCATION_FIELDS:
-				cclog.ComponentDebug("MessageProcessor", "Adding field", data.Value, "->", data.Value)
+				//cclog.ComponentDebug("MessageProcessor", "Adding field", data.Value, "->", data.Value)
 				message.AddField(data.Key, data.Value)
 			case MESSAGE_LOCATION_TAGS:
-				cclog.ComponentDebug("MessageProcessor", "Adding tag", data.Value, "->", data.Value)
+				//cclog.ComponentDebug("MessageProcessor", "Adding tag", data.Value, "->", data.Value)
 				message.AddTag(data.Key, data.Value)
 			case MESSAGE_LOCATION_META:
-				cclog.ComponentDebug("MessageProcessor", "Adding meta", data.Value, "->", data.Value)
+				//cclog.ComponentDebug("MessageProcessor", "Adding meta", data.Value, "->", data.Value)
 				message.AddMeta(data.Key, data.Value)
 			}
 		}
@@ -191,13 +188,13 @@ func normalizeUnits(message lp2.CCMessage) (bool, error) {
 	if in_unit, ok := message.GetMeta("unit"); ok {
 		u := units.NewUnit(in_unit)
 		if u.Valid() {
-			cclog.ComponentDebug("MessageProcessor", "Update unit with", u.Short())
+			//cclog.ComponentDebug("MessageProcessor", "Update unit with", u.Short())
 			message.AddMeta("unit", u.Short())
 		}
 	} else if in_unit, ok := message.GetTag("unit"); ok {
 		u := units.NewUnit(in_unit)
 		if u.Valid() {
-			cclog.ComponentDebug("MessageProcessor", "Update unit with", u.Short())
+			//cclog.ComponentDebug("MessageProcessor", "Update unit with", u.Short())
 			message.AddTag("unit", u.Short())
 		}
 	}
@@ -212,15 +209,15 @@ func changeUnitPrefix(message lp2.CCMessage, params *map[string]interface{}, che
 		}
 		if value.(bool) {
 			newPrefix := units.NewPrefix(n)
-			cclog.ComponentDebug("MessageProcessor", "Condition matches, change to prefix", newPrefix.String())
+			//cclog.ComponentDebug("MessageProcessor", "Condition matches, change to prefix", newPrefix.String())
 			if in_unit, ok := message.GetMeta("unit"); ok && newPrefix != units.InvalidPrefix {
 				u := units.NewUnit(in_unit)
 				if u.Valid() {
-					cclog.ComponentDebug("MessageProcessor", "Input unit", u.Short())
+					//cclog.ComponentDebug("MessageProcessor", "Input unit", u.Short())
 					conv, out_unit := units.GetUnitPrefixFactor(u, newPrefix)
 					if conv != nil && out_unit.Valid() {
 						if val, ok := message.GetField("value"); ok {
-							cclog.ComponentDebug("MessageProcessor", "Update unit with", out_unit.Short())
+							//cclog.ComponentDebug("MessageProcessor", "Update unit with", out_unit.Short())
 							message.AddField("value", conv(val))
 							message.AddMeta("unit", out_unit.Short())
 						}
@@ -230,11 +227,11 @@ func changeUnitPrefix(message lp2.CCMessage, params *map[string]interface{}, che
 			} else if in_unit, ok := message.GetTag("unit"); ok && newPrefix != units.InvalidPrefix {
 				u := units.NewUnit(in_unit)
 				if u.Valid() {
-					cclog.ComponentDebug("MessageProcessor", "Input unit", u.Short())
+					//cclog.ComponentDebug("MessageProcessor", "Input unit", u.Short())
 					conv, out_unit := units.GetUnitPrefixFactor(u, newPrefix)
 					if conv != nil && out_unit.Valid() {
 						if val, ok := message.GetField("value"); ok {
-							cclog.ComponentDebug("MessageProcessor", "Update unit with", out_unit.Short())
+							//cclog.ComponentDebug("MessageProcessor", "Update unit with", out_unit.Short())
 							message.AddField("value", conv(val))
 							message.AddTag("unit", out_unit.Short())
 						}
@@ -255,9 +252,9 @@ func renameMessagesIf(message lp2.CCMessage, params *map[string]interface{}, che
 		}
 		if value.(bool) {
 			old := message.Name()
-			cclog.ComponentDebug("MessageProcessor", "Rename to", n)
+			//cclog.ComponentDebug("MessageProcessor", "Rename to", n)
 			message.SetName(n)
-			cclog.ComponentDebug("MessageProcessor", "Add old name as 'oldname' to meta", old)
+			//cclog.ComponentDebug("MessageProcessor", "Add old name as 'oldname' to meta", old)
 			message.AddMeta("oldname", old)
 		}
 	}