Update rapl collector with powercap limits

* Check creation of CCMessage in NATS receiver

* add derived_values for numastats

* change to ccMessage

* remove vim command artefact

---------

Co-authored-by: Thomas Roehl <thomas.roehl@fau.de>
Co-authored-by: exterr2f <Robert.Externbrink@rub.de>
Co-authored-by: Thomas Gruber <Thomas.Roehl@googlemail.com>

.github/ci-config.json

@@ -1,10 +1,8 @@
 {
-  "sinks-file": ".github/ci-sinks.json",
-  "collectors-file" : ".github/ci-collectors.json",
-  "receivers-file" : ".github/ci-receivers.json",
-  "router-file" : ".github/ci-router.json",
-  "main" : {
-    "interval": "5s",
-    "duration": "1s"
-  }
+  "sinks": ".github/ci-sinks.json",
+  "collectors" : ".github/ci-collectors.json",
+  "receivers" : ".github/ci-receivers.json",
+  "router" : ".github/ci-router.json",
+  "interval": "5s",
+  "duration": "1s"
 }

.github/workflows/runonce.yml

@@ -11,10 +11,10 @@ on:
 jobs:
 
   #
-  # Job build-latest
-  # Build on latest Ubuntu using latest golang version
+  # Job build-1-21
+  # Build on latest Ubuntu using golang version 1.21
   #
-  build-latest:
+  build-1-21:
     runs-on: ubuntu-latest
     steps:
     # See: https://github.com/marketplace/actions/checkout
@@ -29,7 +29,58 @@ jobs:
       uses: actions/setup-go@v5
       with:
         go-version: '1.21'
-        check-latest: true
+
+    - name: Build MetricCollector
+      run: make
+
+    - name: Run MetricCollector once
+      run: ./cc-metric-collector --once --config .github/ci-config.json
+
+  #
+  # Job build-1-22
+  # Build on latest Ubuntu using golang version 1.22
+  #
+  build-1-22:
+    runs-on: ubuntu-latest
+    steps:
+    # See: https://github.com/marketplace/actions/checkout
+    # Checkout git repository and submodules
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    # See: https://github.com/marketplace/actions/setup-go-environment
+    - name: Setup Golang
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.22'
+
+    - name: Build MetricCollector
+      run: make
+
+    - name: Run MetricCollector once
+      run: ./cc-metric-collector --once --config .github/ci-config.json
+
+  #
+  # Job build-1-23
+  # Build on latest Ubuntu using golang version 1.23
+  #
+  build-1-23:
+    runs-on: ubuntu-latest
+    steps:
+    # See: https://github.com/marketplace/actions/checkout
+    # Checkout git repository and submodules
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    # See: https://github.com/marketplace/actions/setup-go-environment
+    - name: Setup Golang
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.23'
 
     - name: Build MetricCollector
       run: make

README.md

@@ -1,17 +1,6 @@
-<!--
----
-title: cc-metric-collector
-description: Metric collecting node agent
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/_index.md
----
--->
-
 # cc-metric-collector
 
-A node agent for measuring, processing and forwarding node level metrics. It is part of the [ClusterCockpit ecosystem](https://clustercockpit.org/docs/overview/).
+A node agent for measuring, processing and forwarding node level metrics. It is part of the [ClusterCockpit ecosystem](./docs/introduction.md).
 
 The metric collector sends (and receives) metric in the [InfluxDB line protocol](https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol/) as it provides flexibility while providing a separation between tags (like index columns in relational databases) and fields (like data columns).
 
@@ -46,8 +35,8 @@ The `interval` defines how often the metrics should be read and send to the sink
 See the component READMEs for their configuration:
 
 * [`collectors`](./collectors/README.md)
-* [`sinks`](https://github.com/ClusterCockpit/cc-lib/blob/main/sinks/README.md)
-* [`receivers`](https://github.com/ClusterCockpit/cc-lib/blob/main/receivers/README.md)
+* [`sinks`](./sinks/README.md)
+* [`receivers`](./receivers/README.md)
 * [`router`](./internal/metricRouter/README.md)
 
 # Installation

cc-metric-collector.go

@@ -7,24 +7,39 @@ import (
 	"os/signal"
 	"syscall"
 
-	"github.com/ClusterCockpit/cc-lib/receivers"
-	"github.com/ClusterCockpit/cc-lib/sinks"
 	"github.com/ClusterCockpit/cc-metric-collector/collectors"
+	"github.com/ClusterCockpit/cc-metric-collector/receivers"
+	"github.com/ClusterCockpit/cc-metric-collector/sinks"
 
 	//	"strings"
 	"sync"
 	"time"
 
-	ccconf "github.com/ClusterCockpit/cc-lib/ccConfig"
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
 	mr "github.com/ClusterCockpit/cc-metric-collector/internal/metricRouter"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	mct "github.com/ClusterCockpit/cc-metric-collector/pkg/multiChanTicker"
 )
 
 type CentralConfigFile struct {
-	Interval string `json:"interval"`
-	Duration string `json:"duration"`
+	Interval            string `json:"interval"`
+	Duration            string `json:"duration"`
+	CollectorConfigFile string `json:"collectors"`
+	RouterConfigFile    string `json:"router"`
+	SinkConfigFile      string `json:"sinks"`
+	ReceiverConfigFile  string `json:"receivers,omitempty"`
+}
+
+func LoadCentralConfiguration(file string, config *CentralConfigFile) error {
+	configFile, err := os.Open(file)
+	if err != nil {
+		cclog.Error(err.Error())
+		return err
+	}
+	defer configFile.Close()
+	jsonParser := json.NewDecoder(configFile)
+	err = jsonParser.Decode(config)
+	return err
 }
 
 type RuntimeConfig struct {
@@ -72,7 +87,7 @@ func ReadCli() map[string]string {
 	cfg := flag.String("config", "./config.json", "Path to configuration file")
 	logfile := flag.String("log", "stderr", "Path for logfile")
 	once := flag.Bool("once", false, "Run all collectors only once")
-	loglevel := flag.String("loglevel", "info", "Set log level")
+	debug := flag.Bool("debug", false, "Activate debug output")
 	flag.Parse()
 	m = make(map[string]string)
 	m["configfile"] = *cfg
@@ -82,7 +97,12 @@ func ReadCli() map[string]string {
 	} else {
 		m["once"] = "false"
 	}
-	m["loglevel"] = *loglevel
+	if *debug {
+		m["debug"] = "true"
+		cclog.SetDebug()
+	} else {
+		m["debug"] = "false"
+	}
 	return m
 }
 
@@ -147,15 +167,8 @@ func mainFunc() int {
 		CliArgs:        ReadCli(),
 	}
 
-	// Set loglevel based on command line input.
-	cclog.Init(rcfg.CliArgs["loglevel"], false)
-
-	// Init ccConfig with configuration file
-	ccconf.Init(rcfg.CliArgs["configfile"])
-
 	// Load and check configuration
-	main := ccconf.GetPackageConfig("main")
-	err = json.Unmarshal(main, &rcfg.ConfigFile)
+	err = LoadCentralConfiguration(rcfg.CliArgs["configfile"], &rcfg.ConfigFile)
 	if err != nil {
 		cclog.Error("Error reading configuration file ", rcfg.CliArgs["configfile"], ": ", err.Error())
 		return 1
@@ -191,41 +204,38 @@ func mainFunc() int {
 		return 1
 	}
 
-	routerConf := ccconf.GetPackageConfig("router")
-	if len(routerConf) == 0 {
+	if len(rcfg.ConfigFile.RouterConfigFile) == 0 {
 		cclog.Error("Metric router configuration file must be set")
 		return 1
 	}
 
-	sinkConf := ccconf.GetPackageConfig("sinks")
-	if len(sinkConf) == 0 {
+	if len(rcfg.ConfigFile.SinkConfigFile) == 0 {
 		cclog.Error("Sink configuration file must be set")
 		return 1
 	}
 
-	collectorConf := ccconf.GetPackageConfig("collectors")
-	if len(collectorConf) == 0 {
+	if len(rcfg.ConfigFile.CollectorConfigFile) == 0 {
 		cclog.Error("Metric collector configuration file must be set")
 		return 1
 	}
 
 	// Set log file
-	// if logfile := rcfg.CliArgs["logfile"]; logfile != "stderr" {
-	// 	cclog.SetOutput(logfile)
-	// }
+	if logfile := rcfg.CliArgs["logfile"]; logfile != "stderr" {
+		cclog.SetOutput(logfile)
+	}
 
 	// Creat new multi channel ticker
 	rcfg.MultiChanTicker = mct.NewTicker(rcfg.Interval)
 
 	// Create new metric router
-	rcfg.MetricRouter, err = mr.New(rcfg.MultiChanTicker, &rcfg.Sync, routerConf)
+	rcfg.MetricRouter, err = mr.New(rcfg.MultiChanTicker, &rcfg.Sync, rcfg.ConfigFile.RouterConfigFile)
 	if err != nil {
 		cclog.Error(err.Error())
 		return 1
 	}
 
 	// Create new sink
-	rcfg.SinkManager, err = sinks.New(&rcfg.Sync, sinkConf)
+	rcfg.SinkManager, err = sinks.New(&rcfg.Sync, rcfg.ConfigFile.SinkConfigFile)
 	if err != nil {
 		cclog.Error(err.Error())
 		return 1
@@ -237,7 +247,7 @@ func mainFunc() int {
 	rcfg.MetricRouter.AddOutput(RouterToSinksChannel)
 
 	// Create new collector manager
-	rcfg.CollectManager, err = collectors.New(rcfg.MultiChanTicker, rcfg.Duration, &rcfg.Sync, collectorConf)
+	rcfg.CollectManager, err = collectors.New(rcfg.MultiChanTicker, rcfg.Duration, &rcfg.Sync, rcfg.ConfigFile.CollectorConfigFile)
 	if err != nil {
 		cclog.Error(err.Error())
 		return 1
@@ -249,9 +259,8 @@ func mainFunc() int {
 	rcfg.MetricRouter.AddCollectorInput(CollectToRouterChannel)
 
 	// Create new receive manager
-	receiveConf := ccconf.GetPackageConfig("receivers")
-	if len(receiveConf) > 0 {
-		rcfg.ReceiveManager, err = receivers.New(&rcfg.Sync, receiveConf)
+	if len(rcfg.ConfigFile.ReceiverConfigFile) > 0 {
+		rcfg.ReceiveManager, err = receivers.New(&rcfg.Sync, rcfg.ConfigFile.ReceiverConfigFile)
 		if err != nil {
 			cclog.Error(err.Error())
 			return 1

collectors/README.md

@@ -1,14 +1,3 @@
-<!--
----
-title: Metric Collectors
-description: Metric collectors for cc-metric-collector
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/_index.md
----
--->
-
 # CCMetric collectors
 
 This folder contains the collectors for the cc-metric-collector.
@@ -34,6 +23,7 @@ In contrast to the configuration files for sinks and receivers, the collectors c
 * [`loadavg`](./loadavgMetric.md)
 * [`netstat`](./netstatMetric.md)
 * [`ibstat`](./infinibandMetric.md)
+* [`ibstat_perfquery`](./infinibandPerfQueryMetric.md)
 * [`tempstat`](./tempMetric.md)
 * [`lustrestat`](./lustreMetric.md)
 * [`likwid`](./likwidMetric.md)
@@ -43,10 +33,8 @@ In contrast to the configuration files for sinks and receivers, the collectors c
 * [`topprocs`](./topprocsMetric.md)
 * [`nfs3stat`](./nfs3Metric.md)
 * [`nfs4stat`](./nfs4Metric.md)
-* [`nfsiostat`](./nfsiostatMetric.md)
 * [`cpufreq`](./cpufreqMetric.md)
 * [`cpufreq_cpuinfo`](./cpufreqCpuinfoMetric.md)
-* [`schedstat`](./schedstatMetric.md)
 * [`numastats`](./numastatsMetric.md)
 * [`gpfs`](./gpfsMetric.md)
 * [`beegfs_meta`](./beegfsmetaMetric.md)
@@ -63,7 +51,7 @@ A collector reads data from any source, parses it to metrics and submits these m
 * `Name() string`: Return the name of the collector
 * `Init(config json.RawMessage) error`: Initializes the collector using the given collector-specific config in JSON. Check if needed files/commands exists, ...
 * `Initialized() bool`: Check if a collector is successfully initialized
-* `Read(duration time.Duration, output chan ccMessage.CCMessage)`: Read, parse and submit data to the `output` channel as [`CCMessage`](https://github.com/ClusterCockpit/cc-lib/blob/main/ccMessage/README.md). If the collector has to measure anything for some duration, use the provided function argument `duration`.
+* `Read(duration time.Duration, output chan ccMetric.CCMetric)`: Read, parse and submit data to the `output` channel as [`CCMetric`](../internal/ccMetric/README.md). If the collector has to measure anything for some duration, use the provided function argument `duration`.
 * `Close()`: Closes down the collector.
 
 It is recommanded to call `setup()` in the `Init()` function.

collectors/beegfsmetaMetric.go

@@ -14,8 +14,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )
 
 const DEFAULT_BEEGFS_CMD = "beegfs-ctl"

collectors/beegfsmetaMetric.md

@@ -1,17 +1,5 @@
-<!--
----
-title: BeeGFS metadata metric collector
-description: Collect metadata clientstats for `BeeGFS on Demand`
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/beegfsmeta.md
----
--->
-
-
 ## `BeeGFS on Demand` collector
-This Collector is to collect `BeeGFS on Demand` (BeeOND) metadata clientstats.
+This Collector is to collect BeeGFS on Demand (BeeOND) metadata clientstats.
 
 ```json
   "beegfs_meta": {

collectors/beegfsstorageMetric.go

@@ -14,8 +14,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )
 
 // Struct for the collector-specific JSON config

collectors/beegfsstorageMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: "BeeGFS on Demand metric collector"
-description: Collect performance metrics for BeeGFS filesystems
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/beegfsstorage.md
----
--->
-
 ## `BeeGFS on Demand` collector
 This Collector is to collect BeeGFS on Demand (BeeOND) storage stats.
 

collectors/collectorManager.go

@@ -2,11 +2,12 @@ package collectors
 
 import (
 	"encoding/json"
+	"os"
 	"sync"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 	mct "github.com/ClusterCockpit/cc-metric-collector/pkg/multiChanTicker"
 )
 
@@ -58,7 +59,7 @@ type collectorManager struct {
 
 // Metric collector manager access functions
 type CollectorManager interface {
-	Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfig json.RawMessage) error
+	Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) error
 	AddOutput(output chan lp.CCMessage)
 	Start()
 	Close()
@@ -71,7 +72,7 @@ type CollectorManager interface {
 // * ticker (from variable ticker)
 // * configuration (read from config file in variable collectConfigFile)
 // Initialization is done for all configured collectors
-func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfig json.RawMessage) error {
+func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) error {
 	cm.collectors = make([]MetricCollector, 0)
 	cm.serial = make([]MetricCollector, 0)
 	cm.output = nil
@@ -80,7 +81,15 @@ func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Durat
 	cm.ticker = ticker
 	cm.duration = duration
 
-	err := json.Unmarshal(collectConfig, &cm.config)
+	// Read collector config file
+	configFile, err := os.Open(collectConfigFile)
+	if err != nil {
+		cclog.Error(err.Error())
+		return err
+	}
+	defer configFile.Close()
+	jsonParser := json.NewDecoder(configFile)
+	err = jsonParser.Decode(&cm.config)
 	if err != nil {
 		cclog.Error(err.Error())
 		return err
@@ -191,9 +200,9 @@ func (cm *collectorManager) Close() {
 }
 
 // New creates a new initialized metric collector manager
-func New(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfig json.RawMessage) (CollectorManager, error) {
+func New(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) (CollectorManager, error) {
 	cm := new(collectorManager)
-	err := cm.Init(ticker, duration, wg, collectConfig)
+	err := cm.Init(ticker, duration, wg, collectConfigFile)
 	if err != nil {
 		return nil, err
 	}

collectors/cpufreqCpuinfoMetric.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 // CPUFreqCollector

collectors/cpufreqCpuinfoMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: CPU frequency metric collector through cpuinfo
-description: Collect the CPU frequency from `/proc/cpuinfo`
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/cpufreq_cpuinfo.md
----
--->
-
 ## `cpufreq_cpuinfo` collector
 
 ```json

collectors/cpufreqMetric.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	"github.com/ClusterCockpit/cc-metric-collector/pkg/ccTopology"
 	"golang.org/x/sys/unix"
 )

collectors/cpufreqMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: CPU frequency metric collector through sysfs
-description: Collect the CPU frequency metrics from `/sys/.../cpu/.../cpufreq`
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/cpufreq.md
----
--->
-
 ## `cpufreq_cpuinfo` collector
 
 ```json

collectors/cpustatMetric.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 	sysconf "github.com/tklauser/go-sysconf"
 )
 

collectors/cpustatMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: CPU usage metric collector
-description: Collect CPU metrics from `/proc/stat`
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/cpustat.md
----
--->
-
 
 ## `cpustat` collector
 

collectors/customCmdMetric.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	influx "github.com/influxdata/line-protocol"
 )
 

collectors/customCmdMetric.md

@@ -1,13 +1,3 @@
-<!--
----
-title: CustomCommand metric collector
-description: Collect messages from custom command or files
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/customcmd.md
----
--->
 
 ## `customcmd` collector
 

collectors/diskstatMetric.go

@@ -8,8 +8,8 @@ import (
 	"syscall"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 const MOUNTFILE = `/proc/self/mounts`

collectors/diskstatMetric.md

@@ -1,13 +1,3 @@
-<!--
----
-title: Disk usage statistics metric collector
-description: Collect metrics for various filesystems from `/proc/self/mounts`
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/diskstat.md
----
--->
 
 ## `diskstat` collector
 

collectors/gpfsMetric.go

@@ -13,8 +13,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )
 
 const DEFAULT_GPFS_CMD = "mmpmon"

collectors/gpfsMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: GPFS collector
-description: Collect infos about GPFS filesystems
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/gpfs.md
----
--->
-
 ## `gpfs` collector
 
 ```json

collectors/infinibandMetric.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"os"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 	"golang.org/x/sys/unix"
 
 	"encoding/json"

collectors/infinibandMetric.md

@@ -1,13 +1,3 @@
-<!--
----
-title: InfiniBand Metric collector
-description: Collect metrics for InfiniBand devices
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/infiniband.md
----
--->
 
 ## `ibstat` collector
 

collectors/iostatMetric.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 // Konstante für den Pfad zu /proc/diskstats

collectors/iostatMetric.md

@@ -1,13 +1,3 @@
-<!--
----
-title: IOStat Metric collector
-description: Collect metrics from `/proc/diskstats`
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/iostat.md
----
--->
 
 ## `iostat` collector
 

collectors/ipmiMetric.go

@@ -13,8 +13,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )
 
 const IPMISENSORS_PATH = `ipmi-sensors`

collectors/ipmiMetric.md

@@ -1,13 +1,3 @@
-<!--
----
-title: IPMI Metric collector
-description: Collect metrics using ipmitool or ipmi-sensors
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/ipmi.md
----
--->
 
 ## `ipmistat` collector
 

collectors/likwidMetric.go

@@ -24,9 +24,9 @@ import (
 	"time"
 	"unsafe"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	agg "github.com/ClusterCockpit/cc-metric-collector/internal/metricAggregator"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 	topo "github.com/ClusterCockpit/cc-metric-collector/pkg/ccTopology"
 	"github.com/NVIDIA/go-nvml/pkg/dl"
 	"github.com/fsnotify/fsnotify"
@@ -190,8 +190,12 @@ func getBaseFreq() float64 {
 	}
 
 	if math.IsNaN(freq) {
-		C.timer_init()
-		freq = float64(C.timer_getCycleClock()) / 1e3
+		C.power_init(0)
+		info := C.get_powerInfo()
+		if float64(info.baseFrequency) != 0 {
+			freq = float64(info.baseFrequency)
+		}
+		C.power_finalize()
 	}
 	return freq * 1e3
 }

collectors/likwidMetric.md

@@ -1,13 +1,3 @@
-<!--
----
-title: LIKWID collector
-description: Collect hardware performance events and metrics using LIKWID
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/likwid.md
----
--->
 
 ## `likwid` collector
 

collectors/loadavgMetric.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 // LoadavgCollector collects:

collectors/loadavgMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: Load average metric collector
-description: Collect metrics from `/proc/loadavg`
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/loadavg.md
----
--->
-
 
 ## `loadavg` collector
 

collectors/lustreMetric.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )
 
 const LUSTRE_SYSFS = `/sys/fs/lustre`

collectors/lustreMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: Lustre filesystem metric collector
-description: Collect metrics for Lustre filesystems
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/lustre.md
----
--->
-
 
 ## `lustrestat` collector
 

collectors/memstatMetric.go

@@ -12,8 +12,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 const MEMSTATFILE = "/proc/meminfo"

collectors/memstatMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: Memory statistics metric collector
-description: Collect metrics from `/proc/meminfo`
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/memstat.md
----
--->
-
 
 ## `memstat` collector
 

collectors/metricCollector.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"time"
 
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )
 
 type MetricCollector interface {
@@ -14,7 +14,7 @@ type MetricCollector interface {
 	Initialized() bool                 // Is metric collector initialized?
 	Parallel() bool
 	Read(duration time.Duration, output chan lp.CCMessage) // Read metrics from metric collector
-	Close()                                                // Close / finish metric collector
+	Close()                                               // Close / finish metric collector
 }
 
 type metricCollector struct {

collectors/netstatMetric.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 const NETSTATFILE = "/proc/net/dev"

collectors/netstatMetric.md

@@ -1,13 +1,3 @@
-<!--
----
-title: Network device metric collector
-description: Collect metrics for network devices through procfs
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/netstat.md
----
--->
 
 ## `netstat` collector
 

collectors/nfs3Metric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: NFS network filesystem (v3) metric collector
-description: Collect metrics for NFS network filesystems in version 3
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/nfs3.md
----
--->
-
 
 ## `nfs3stat` collector
 

collectors/nfs4Metric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: NFS network filesystem (v4) metric collector
-description: Collect metrics for NFS network filesystems in version 4
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/nfs4.md
----
--->
-
 
 ## `nfs4stat` collector
 

collectors/nfsMetric.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"time"
 
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )
 
 // First part contains the code for the general NfsCollector.

collectors/nfsiostatMetric.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 // These are the fields we read from the JSON configuration
@@ -171,7 +171,7 @@ func (m *NfsIOStatCollector) Read(interval time.Duration, output chan lp.CCMessa
 			}
 		}
 		if !found {
-			delete(m.data, mntpoint)
+			m.data[mntpoint] = nil
 		}
 	}
 }

collectors/nfsiostatMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: NFS network filesystem metrics from procfs
-description: Collect NFS network filesystem metrics for mounts from `/proc/self/mountstats`
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/nfsio.md
----
--->
-
 ## `nfsiostat` collector
 
 ```json

collectors/numastatsMetric.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 type NUMAStatsCollectorConfig struct {

collectors/numastatsMetric.md

@@ -1,13 +1,3 @@
-<!--
----
-title: NUMAStat collector
-description: Collect infos about NUMA domains
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/numastat.md
----
--->
 
 ## `numastat` collector
 

collectors/nvidiaMetric.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 	"github.com/NVIDIA/go-nvml/pkg/nvml"
 )
 

collectors/nvidiaMetric.md

@@ -1,13 +1,3 @@
-<!--
----
-title: "Nvidia NVML metric collector"
-description: Collect metrics for Nvidia GPUs using the NVML
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/nvidia.md
----
--->
 
 ## `nvidia` collector
 

collectors/raplMetric.go

@@ -9,20 +9,29 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 // running average power limit (RAPL) monitoring attributes for a zone
+// Only for Intel systems
+
 type RAPLZoneInfo struct {
+	energy            int64     // current reading of the energy counter in micro joules
+	maxEnergyRange    int64     // Range of the above energy counter in micro-joules
+	energyTimestamp   time.Time // timestamp when energy counter was read
+	energyFilepath    string    // path to a file containing the zones current energy counter in micro joules
+	shortTermFilepath string    // path to short term power limit
+	longTermFilepath  string    // path to long term power limit
+	enabledFilepath   string    // path to check whether limits are enabled
+	name              string
+
 	// tags describing the RAPL zone:
 	// * zone_name, subzone_name: e.g. psys, dram, core, uncore, package-0
 	// * zone_id: e.g. 0:1 (zone 0 sub zone 1)
-	tags            map[string]string
-	energyFilepath  string    // path to a file containing the zones current energy counter in micro joules
-	energy          int64     // current reading of the energy counter in micro joules
-	energyTimestamp time.Time // timestamp when energy counter was read
-	maxEnergyRange  int64     // Range of the above energy counter in micro-joules
+	// type=socket for dram, core, uncore, package-* and type=node for psys
+	// type-id=socket id
+	tags map[string]string
 }
 
 type RAPLCollector struct {
@@ -33,12 +42,40 @@ type RAPLCollector struct {
 		// * 0:1 for zone 0 subzone 1
 		ExcludeByID []string `json:"exclude_device_by_id,omitempty"`
 		// Exclude names for RAPL zones, e.g. psys, dram, core, uncore, package-0
-		ExcludeByName []string `json:"exclude_device_by_name,omitempty"`
+		ExcludeByName     []string `json:"exclude_device_by_name,omitempty"`
+		SkipEnergyReading bool     `json:"skip_energy_reading,omitempty"`
+		SkipLimitsReading bool     `json:"skip_limits_reading,omitempty"`
+		OnlyEnabledLimits bool     `json:"only_enabled_limits,omitempty"`
 	}
-	RAPLZoneInfo []RAPLZoneInfo
+	raplZoneInfo []RAPLZoneInfo
 	meta         map[string]string // default meta information
 }
 
+// Get the path to the power limit file for zone selectable by limit name
+// Common limit names for Intel systems are
+// - long_term
+// - short_term
+// Does not support AMD as AMD systems do not provide the power limits
+// through sysfs
+func ZoneLimitFile(folder string, limit_name string) string {
+	nameGlob := filepath.Join(folder, "constraint_*_name")
+	candidates, err := filepath.Glob(nameGlob)
+	if err == nil {
+		for _, c := range candidates {
+			if v, err := os.ReadFile(c); err == nil {
+				if strings.TrimSpace(string(v)) == limit_name {
+					var i int
+					n, err := fmt.Sscanf(filepath.Base(c), "constraint_%d_name", &i)
+					if err == nil && n == 1 {
+						return filepath.Join(folder, fmt.Sprintf("constraint_%d_power_limit_uw", i))
+					}
+				}
+			}
+		}
+	}
+	return ""
+}
+
 // Init initializes the running average power limit (RAPL) collector
 func (m *RAPLCollector) Init(config json.RawMessage) error {
 
@@ -58,6 +95,9 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 	}
 
 	// Read in the JSON configuration
+	m.config.SkipEnergyReading = false
+	m.config.SkipLimitsReading = false
+	m.config.OnlyEnabledLimits = true
 	if len(config) > 0 {
 		err = json.Unmarshal(config, &m.config)
 		if err != nil {
@@ -83,50 +123,62 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 	// readZoneInfo reads RAPL monitoring attributes for a zone given by zonePath
 	// See: https://www.kernel.org/doc/html/latest/power/powercap/powercap.html#monitoring-attributes
 	readZoneInfo := func(zonePath string) (z struct {
-		name            string    // zones name e.g. psys, dram, core, uncore, package-0
-		energyFilepath  string    // path to a file containing the zones current energy counter in micro joules
-		energy          int64     // current reading of the energy counter in micro joules
-		energyTimestamp time.Time // timestamp when energy counter was read
-		maxEnergyRange  int64     // Range of the above energy counter in micro-joules
-		ok              bool      // Are all information available?
+		name              string    // zones name e.g. psys, dram, core, uncore, package-0
+		energyFilepath    string    // path to a file containing the zones current energy counter in micro joules
+		energy            int64     // current reading of the energy counter in micro joules
+		energyTimestamp   time.Time // timestamp when energy counter was read
+		maxEnergyRange    int64     // Range of the above energy counter in micro-joules
+		shortTermFilepath string
+		longTermFilepath  string
+		enabledFilepath   string
 	}) {
 		// zones name e.g. psys, dram, core, uncore, package-0
-		foundName := false
+
 		if v, err :=
 			os.ReadFile(
 				filepath.Join(zonePath, "name")); err == nil {
-			foundName = true
 			z.name = strings.TrimSpace(string(v))
 		}
 
-		// path to a file containing the zones current energy counter in micro joules
-		z.energyFilepath = filepath.Join(zonePath, "energy_uj")
-
-		// current reading of the energy counter in micro joules
-		foundEnergy := false
-		if v, err := os.ReadFile(z.energyFilepath); err == nil {
-			// timestamp when energy counter was read
-			z.energyTimestamp = time.Now()
-			if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
-				foundEnergy = true
-				z.energy = i
+		if os.Geteuid() == 0 && (!m.config.SkipEnergyReading) {
+			// path to a file containing the zones current energy counter in micro joules
+			z.energyFilepath = filepath.Join(zonePath, "energy_uj")
+			// current reading of the energy counter in micro joules
+			if v, err := os.ReadFile(z.energyFilepath); err == nil {
+				if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
+					z.energy = i
+					// timestamp when energy counter was read
+					z.energyTimestamp = time.Now()
+				}
+			} else {
+				cclog.ComponentError(m.name, "Cannot read energy file for ", z.name, ":", err.Error())
 			}
+			// Range of the above energy counter in micro-joules
+			if v, err :=
+				os.ReadFile(
+					filepath.Join(zonePath, "max_energy_range_uj")); err == nil {
+				if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
+					z.maxEnergyRange = i
+				}
+			}
+		} else {
+			cclog.ComponentInfo(m.name, "Energy readings for", zonePath, "disabled")
 		}
 
-		// Range of the above energy counter in micro-joules
-		foundMaxEnergyRange := false
-		if v, err :=
-			os.ReadFile(
-				filepath.Join(zonePath, "max_energy_range_uj")); err == nil {
-			if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
-				foundMaxEnergyRange = true
-				z.maxEnergyRange = i
+		if !m.config.SkipLimitsReading {
+			z.shortTermFilepath = ZoneLimitFile(zonePath, "short_term")
+			if _, err := os.Stat(z.shortTermFilepath); err != nil {
+				z.shortTermFilepath = ""
 			}
+			z.longTermFilepath = ZoneLimitFile(zonePath, "long_term")
+			if _, err := os.Stat(z.longTermFilepath); err != nil {
+				z.longTermFilepath = ""
+			}
+			z.enabledFilepath = filepath.Join(zonePath, "enabled")
+		} else {
+			cclog.ComponentInfo(m.name, "Power limit readings for", zonePath, "disabled")
 		}
 
-		// Are all information available?
-		z.ok = foundName && foundEnergy && foundMaxEnergyRange
-
 		return
 	}
 
@@ -143,25 +195,42 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 
 	for _, zonePath := range zonesPath {
 		zoneID := strings.TrimPrefix(zonePath, zonePrefix)
+		zonetags := make(map[string]string)
+
 		z := readZoneInfo(zonePath)
-		if z.ok &&
-			!isIDExcluded[zoneID] &&
+		if !isIDExcluded[zoneID] &&
 			!isNameExcluded[z.name] {
 
+			si := RAPLZoneInfo{
+				tags:              make(map[string]string),
+				energyFilepath:    z.energyFilepath,
+				energy:            z.energy,
+				energyTimestamp:   z.energyTimestamp,
+				maxEnergyRange:    z.maxEnergyRange,
+				shortTermFilepath: z.shortTermFilepath,
+				longTermFilepath:  z.longTermFilepath,
+				enabledFilepath:   z.enabledFilepath,
+				name:              z.name,
+			}
+			si.tags["type"] = "node"
+			si.tags["type-id"] = "0"
+			var pid int = 0
+			if strings.HasPrefix(z.name, "package-") {
+				n, err := fmt.Sscanf(z.name, "package-%d", &pid)
+				if err == nil && n == 1 {
+					si.tags["type-id"] = fmt.Sprintf("%d", pid)
+					si.tags["type"] = "socket"
+				}
+				si.name = "pkg"
+			}
 			// Add RAPL monitoring attributes for a zone
-			m.RAPLZoneInfo =
-				append(
-					m.RAPLZoneInfo,
-					RAPLZoneInfo{
-						tags: map[string]string{
-							"id":        zoneID,
-							"zone_name": z.name,
-						},
-						energyFilepath:  z.energyFilepath,
-						energy:          z.energy,
-						energyTimestamp: z.energyTimestamp,
-						maxEnergyRange:  z.maxEnergyRange,
-					})
+			if _, ok1 := si.tags["type"]; ok1 {
+				if _, ok2 := si.tags["type-id"]; ok2 {
+					m.raplZoneInfo = append(m.raplZoneInfo, si)
+					zonetags["type"] = si.tags["type"]
+					zonetags["type-id"] = si.tags["type-id"]
+				}
+			}
 		}
 
 		// find all sub zones for the given zone
@@ -174,29 +243,32 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 		for _, subZonePath := range subZonesPath {
 			subZoneID := strings.TrimPrefix(subZonePath, subZonePrefix)
 			sz := readZoneInfo(subZonePath)
+
 			if len(zoneID) > 0 && len(z.name) > 0 &&
-				sz.ok &&
 				!isIDExcluded[zoneID+":"+subZoneID] &&
 				!isNameExcluded[sz.name] {
-				m.RAPLZoneInfo =
-					append(
-						m.RAPLZoneInfo,
-						RAPLZoneInfo{
-							tags: map[string]string{
-								"id":            zoneID + ":" + subZoneID,
-								"zone_name":     z.name,
-								"sub_zone_name": sz.name,
-							},
-							energyFilepath:  sz.energyFilepath,
-							energy:          sz.energy,
-							energyTimestamp: sz.energyTimestamp,
-							maxEnergyRange:  sz.maxEnergyRange,
-						})
+
+				si := RAPLZoneInfo{
+					tags:              zonetags,
+					energyFilepath:    sz.energyFilepath,
+					energy:            sz.energy,
+					energyTimestamp:   sz.energyTimestamp,
+					maxEnergyRange:    sz.maxEnergyRange,
+					shortTermFilepath: sz.shortTermFilepath,
+					longTermFilepath:  sz.longTermFilepath,
+					enabledFilepath:   sz.enabledFilepath,
+					name:              sz.name,
+				}
+				if _, ok1 := si.tags["type"]; ok1 {
+					if _, ok2 := si.tags["type-id"]; ok2 {
+						m.raplZoneInfo = append(m.raplZoneInfo, si)
+					}
+				}
 			}
 		}
 	}
 
-	if m.RAPLZoneInfo == nil {
+	if m.raplZoneInfo == nil {
 		return fmt.Errorf("no running average power limit (RAPL) device found in %s", controlTypePath)
 
 	}
@@ -205,7 +277,7 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 	cclog.ComponentDebug(
 		m.name,
 		"initialized",
-		len(m.RAPLZoneInfo),
+		len(m.raplZoneInfo),
 		"zones with running average power limit (RAPL) monitoring attributes")
 	m.init = true
 
@@ -216,40 +288,89 @@ func (m *RAPLCollector) Init(config json.RawMessage) error {
 // See: https://www.kernel.org/doc/html/latest/power/powercap/powercap.html#monitoring-attributes
 func (m *RAPLCollector) Read(interval time.Duration, output chan lp.CCMessage) {
 
-	for i := range m.RAPLZoneInfo {
-		p := &m.RAPLZoneInfo[i]
+	for i := range m.raplZoneInfo {
+		p := &m.raplZoneInfo[i]
 
-		// Read current value of the energy counter in micro joules
-		if v, err := os.ReadFile(p.energyFilepath); err == nil {
-			energyTimestamp := time.Now()
-			if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
-				energy := i
+		if os.Geteuid() == 0 && (!m.config.SkipEnergyReading) {
+			// Read current value of the energy counter in micro joules
+			if v, err := os.ReadFile(p.energyFilepath); err == nil {
+				energyTimestamp := time.Now()
+				if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
+					energy := i
 
-				// Compute average power (Δ energy / Δ time)
-				energyDiff := energy - p.energy
-				if energyDiff < 0 {
-					// Handle overflow:
-					// ( p.maxEnergyRange - p.energy ) + energy
-					// = p.maxEnergyRange + ( energy - p.energy )
-					// = p.maxEnergyRange + diffEnergy
-					energyDiff += p.maxEnergyRange
+					// Compute average power (Δ energy / Δ time)
+					energyDiff := energy - p.energy
+					if energyDiff < 0 {
+						// Handle overflow:
+						// ( p.maxEnergyRange - p.energy ) + energy
+						// = p.maxEnergyRange + ( energy - p.energy )
+						// = p.maxEnergyRange + diffEnergy
+						energyDiff += p.maxEnergyRange
+					}
+					timeDiff := energyTimestamp.Sub(p.energyTimestamp)
+					averagePower := float64(energyDiff) / float64(timeDiff.Microseconds())
+
+					y, err := lp.NewMetric(
+						fmt.Sprintf("rapl_%s_average_power", p.name),
+						p.tags,
+						m.meta,
+						averagePower,
+						energyTimestamp)
+					if err == nil {
+						output <- y
+					}
+
+					e, err := lp.NewMetric(
+						fmt.Sprintf("rapl_%s_energy", p.name),
+						p.tags,
+						m.meta,
+						float64(energyDiff)*1e-3,
+						energyTimestamp)
+					if err == nil {
+						e.AddMeta("unit", "Joules")
+						output <- e
+					}
+
+					// Save current energy counter state
+					p.energy = energy
+					p.energyTimestamp = energyTimestamp
 				}
-				timeDiff := energyTimestamp.Sub(p.energyTimestamp)
-				averagePower := float64(energyDiff) / float64(timeDiff.Microseconds())
-
-				y, err := lp.NewMessage(
-					"rapl_average_power",
-					p.tags,
-					m.meta,
-					map[string]interface{}{"value": averagePower},
-					energyTimestamp)
-				if err == nil {
-					output <- y
+			}
+		}
+		// https://www.kernel.org/doc/html/latest/power/powercap/powercap.html#constraints
+		if !m.config.SkipLimitsReading {
+			skip := false
+			if m.config.OnlyEnabledLimits {
+				if v, err := os.ReadFile(p.enabledFilepath); err == nil {
+					if strings.TrimSpace(string(v)) == "0" {
+						skip = true
+					}
+				}
+			}
+			if !skip {
+				if len(p.shortTermFilepath) > 0 {
+					if v, err := os.ReadFile(p.shortTermFilepath); err == nil {
+						if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
+							name := fmt.Sprintf("rapl_%s_limit_short_term", p.name)
+							y, err := lp.NewMetric(name, p.tags, m.meta, i/1e6, time.Now())
+							if err == nil {
+								output <- y
+							}
+						}
+					}
 				}
 
-				// Save current energy counter state
-				p.energy = energy
-				p.energyTimestamp = energyTimestamp
+				if len(p.longTermFilepath) > 0 {
+					if v, err := os.ReadFile(p.longTermFilepath); err == nil {
+						if i, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil {
+							name := fmt.Sprintf("rapl_%s_limit_long_term", p.name)
+							y, err := lp.NewMetric(name, p.tags, m.meta, i/1e6, time.Now())
+							if err == nil {
+								output <- y
+							}
+						}
+					}
+				}
 			}
 		}
 	}

collectors/raplMetric.md

@@ -1,26 +1,23 @@
-<!--
----
-title: RAPL metric collector
-description: Collect energy data through the RAPL sysfs interface
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/rapl.md
----
--->
-
 ## `rapl` collector
 
-This collector reads running average power limit (RAPL) monitoring attributes to compute average power consumption metrics. See <https://www.kernel.org/doc/html/latest/power/powercap/powercap.html#monitoring-attributes>.
-
-The Likwid metric collector provides similar functionality.
+This collector reads running average power limit (RAPL) monitoring attributes to compute average power consumption metrics. See <https://www.kernel.org/doc/html/latest/power/powercap/powercap.html>.
 
 ```json
   "rapl": {
     "exclude_device_by_id": ["0:1", "0:2"],
-    "exclude_device_by_name": ["psys"]
+    "exclude_device_by_name": ["psys"],
+    "skip_energy_reading": false,
+    "skip_limits_reading": false,
+    "only_enabled_limits": true
   }
 ```
 
 Metrics:
-* `rapl_average_power`: average power consumption in Watt. The average is computed over the entire runtime from the last measurement to the current measurement
+* `rapl_<domain>_average_power`: average power consumption in Watt. The average is computed over the entire runtime from the last measurement to the current measurement
+* `rapl_<domain>_energy`: Difference from the last measurement
+* `rapl_<domain>_limit_short_term`: Short term powercap setting for the domain
+* `rapl_<domain>_limit_long_term`: Long term powercap setting for the domain
+
+Only the `rapl_<domain>_average_power` and `rapl_<domain>_energy` metrics require root-permissions. The limits can be read as user. Some domains have limits available but they are not enabled. By default, only enabled domain limits are collected.
+
+Energy and power measurments can also be done with the Likwid metric collector.

collectors/rocmsmiMetric.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	"github.com/ClusterCockpit/go-rocm-smi/pkg/rocm_smi"
 )
 

collectors/rocmsmiMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: "ROCm SMI metric collector"
-description: Collect metrics for AMD GPUs using the SMI library
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/rocmsmi.md
----
--->
-
 
 ## `rocm_smi` collector
 

collectors/sampleMetric.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 // These are the fields we read from the JSON configuration

collectors/sampleTimerMetric.go

@@ -5,8 +5,8 @@ import (
 	"sync"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 // These are the fields we read from the JSON configuration

collectors/schedstatMetric.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )
 
 const SCHEDSTATFILE = `/proc/schedstat`

collectors/schedstatMetric.md

@@ -1,13 +1,3 @@
-<!--
----
-title: SchedStat Metric collector
-description: Collect metrics from `/proc/schedstat`
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/schedstat.md
----
--->
 
 ## `schedstat` collector
 ```json

collectors/selfMetric.go

@@ -6,8 +6,8 @@ import (
 	"syscall"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )
 
 type SelfCollectorConfig struct {

collectors/selfMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: Self-monitoring metric collector
-description: Collect metrics from the execution of cc-metric-collector itself
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/self.md
----
--->
-
 ## `self` collector
 
 ```json

collectors/tempMetric.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )
 
 // See: https://www.kernel.org/doc/html/latest/hwmon/sysfs-interface.html

collectors/tempMetric.md

@@ -1,14 +1,3 @@
-<!--
----
-title: Temperature metric collector
-description: Collect thermal metrics from `/sys/class/hwmon/*`
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/temp.md
----
--->
-
 
 ## `tempstat` collector
 

collectors/topprocsMetric.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 )
 
 const MAX_NUM_PROCS = 10

collectors/topprocsMetric.md

@@ -1,15 +1,3 @@
-<!--
----
-title: TopProcs collector
-description: Collect infos about most CPU-consuming processes
-categories: [cc-metric-collector]
-tags: ['Admin']
-weight: 2
-hugo_path: docs/reference/cc-metric-collector/collectors/topprocs.md
----
--->
-
-
 
 ## `topprocs` collector
 

config.json

@@ -1,10 +1,8 @@
 {
-  "sinks-file": "./sinks.json",
-  "collectors-file" : "./collectors.json",
-  "receivers-file" : "./receivers.json",
-  "router-file" : "./router.json",
-  "main" : {
-    "interval": "10s",
-    "duration": "1s"
-  }
+  "sinks": "./sinks.json",
+  "collectors" : "./collectors.json",
+  "receivers" : "./receivers.json",
+  "router" : "./router.json",
+  "interval": "10s",
+  "duration": "1s"
 }

go.mod

@@ -1,14 +1,17 @@
 module github.com/ClusterCockpit/cc-metric-collector
 
-go 1.23.4
+go 1.23.0
 
-toolchain go1.23.7
+toolchain go1.23.2
 
 require (
-	github.com/ClusterCockpit/cc-lib v0.1.1
+	github.com/ClusterCockpit/cc-energy-manager v0.0.0-20240709142550-dd446f7ab900
+	github.com/ClusterCockpit/cc-lib v0.1.0-beta.1
+	github.com/ClusterCockpit/cc-units v0.4.0
 	github.com/ClusterCockpit/go-rocm-smi v0.3.0
 	github.com/NVIDIA/go-nvml v0.12.0-2
 	github.com/PaesslerAG/gval v1.2.2
+	github.com/expr-lang/expr v1.16.9
 	github.com/fsnotify/fsnotify v1.7.0
 	github.com/gorilla/mux v1.8.1
 	github.com/influxdata/influxdb-client-go/v2 v2.14.0
@@ -20,16 +23,14 @@ require (
 	github.com/tklauser/go-sysconf v0.3.13
 	golang.design/x/thread v0.0.0-20210122121316-335e9adffdf1
 	golang.org/x/exp v0.0.0-20250215185904-eff6e970281f
-	golang.org/x/sys v0.30.0
+	golang.org/x/sys v0.28.0
 )
 
 require (
 	github.com/ClusterCockpit/cc-backend v1.4.2 // indirect
-	github.com/ClusterCockpit/cc-units v0.4.0 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/expr-lang/expr v1.17.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@@ -42,7 +43,7 @@ require (
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/tklauser/numcpus v0.7.0 // indirect
-	golang.org/x/crypto v0.35.0 // indirect
-	golang.org/x/net v0.36.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
+	golang.org/x/net v0.31.0 // indirect
 	google.golang.org/protobuf v1.35.2 // indirect
 )

go.sum

@@ -1,9 +1,9 @@
 github.com/ClusterCockpit/cc-backend v1.4.2 h1:kTOzqkh9N0564N9nqQThnSs7TAfg8RLgvSm00e5HtIc=
 github.com/ClusterCockpit/cc-backend v1.4.2/go.mod h1:g8TNHXe4AXej26snu2//jO3mUF980elT93iV/k11O/c=
+github.com/ClusterCockpit/cc-energy-manager v0.0.0-20240709142550-dd446f7ab900 h1:6+WNav16uWTEDC09hkZKEHfBhtc91p/ZcjgCtyntuIg=
+github.com/ClusterCockpit/cc-energy-manager v0.0.0-20240709142550-dd446f7ab900/go.mod h1:EbYeC5t+Y0kW1Q1pP2n9zMqbeYEJITG8YGvAUihXVn4=
 github.com/ClusterCockpit/cc-lib v0.1.0-beta.1 h1:dz9j0g2cod8+SMDjuoIY6ISpiHHeekhX6yQaeiwiwJw=
 github.com/ClusterCockpit/cc-lib v0.1.0-beta.1/go.mod h1:kXMskla1i5ZSfXW0vVRIHgGeXMU5zu2PzYOYnUaOr80=
-github.com/ClusterCockpit/cc-lib v0.1.1 h1:AXZWYUzgTaE/WdxLNSWPR7FJoA5WlzvYZxw4gIw3gNw=
-github.com/ClusterCockpit/cc-lib v0.1.1/go.mod h1:SHKcWW/+kN+pcofAtHJFxvmx1FV0VIJuQv5PuT0HDcc=
 github.com/ClusterCockpit/cc-units v0.4.0 h1:zP5DOu99GmErW0tCDf0gcLrlWt42RQ9dpoONEOh4cI0=
 github.com/ClusterCockpit/cc-units v0.4.0/go.mod h1:3S3PAhAayS3pbgcT4q9Vn9VJw22Op51X0YimtG77zBw=
 github.com/ClusterCockpit/go-rocm-smi v0.3.0 h1:1qZnSpG7/NyLtc7AjqnUL9Jb8xtqG1nMVgp69rJfaR8=
@@ -29,8 +29,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI=
 github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
-github.com/expr-lang/expr v1.17.0 h1:+vpszOyzKLQXC9VF+wA8cVA0tlA984/Wabc/1hF9Whg=
-github.com/expr-lang/expr v1.17.0/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
 github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
 github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
 github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
@@ -115,19 +113,13 @@ golang.design/x/thread v0.0.0-20210122121316-335e9adffdf1 h1:P7S/GeHBAFEZIYp0ePP
 golang.design/x/thread v0.0.0-20210122121316-335e9adffdf1/go.mod h1:9CWpnTUmlQkfdpdutA1nNf4iE5lAVt3QZOu0Z6hahBE=
 golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
-golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
 golang.org/x/exp v0.0.0-20250215185904-eff6e970281f h1:oFMYAjX0867ZD2jcNiLBrI9BdpmEkvPyi5YrBGXbamg=
 golang.org/x/exp v0.0.0-20250215185904-eff6e970281f/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
 golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
 golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
-golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA=
-golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I=
 golang.org/x/sys v0.0.0-20210122093101-04d7465088b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
 google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=

internal/metricAggregator/README.md

@@ -1,14 +1,3 @@
-<!--
----
-title: Metric Aggregator
-description: Subsystem for evaluating expressions on metrics (deprecated)
-categories: [cc-metric-collector]
-tags: ['Developer']
-weight: 1
-hugo_path: docs/reference/cc-metric-collector/internal/metricaggregator/_index.md
----
--->
-
 # The MetricAggregator
 
 In some cases, further combination of metrics or raw values is required. For that strings like `foo + 1` with runtime dependent `foo` need to be evaluated. The MetricAggregator relies on the [`gval`](https://github.com/PaesslerAG/gval) Golang package to perform all expression evaluation. The `gval` package provides the basic arithmetic operations but the MetricAggregator defines additional ones.

internal/metricAggregator/metricAggregator.go

@@ -9,9 +9,9 @@ import (
 	"sync"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	topo "github.com/ClusterCockpit/cc-metric-collector/pkg/ccTopology"
 
 	"github.com/PaesslerAG/gval"

internal/metricRouter/README.md

@@ -1,22 +1,11 @@
-<!--
----
-title: Message Router
-description: Routing component inside cc-metric-collector
-categories: [cc-metric-collector]
-tags: ['Developer']
-weight: 1
-hugo_path: docs/reference/cc-metric-collector/internal/metricrouter/_index.md
----
--->
-
 # CC Metric Router
 
-The CCMetric router sits in between the collectors and the sinks and can be used to add and remove tags to/from traversing [CCMessages](https://pkg.go.dev/github.com/ClusterCockpit/cc-lib/ccMessage).
+The CCMetric router sits in between the collectors and the sinks and can be used to add and remove tags to/from traversing [CCMessages](https://pkg.go.dev/github.com/ClusterCockpit/cc-energy-manager@v0.0.0-20240919152819-92a17f2da4f7/pkg/cc-message.
 
 
 # Configuration
 
-**Note**: Use the [message processor configuration](https://github.com/ClusterCockpit/cc-lib/blob/main/messageProcessor/README.md) with option `process_messages`.
+**Note**: Use the [message processor configuration](../../pkg/messageProcessor/README.md) with option `process_messages`.
 
 ```json
 {
@@ -80,7 +69,7 @@ The CCMetric router sits in between the collectors and the sinks and can be used
 
 There are three main options `add_tags`, `delete_tags` and `interval_timestamp`. `add_tags` and `delete_tags` are lists consisting of dicts with `key`, `value` and `if`. The `value` can be omitted in the `delete_tags` part as it only uses the `key` for removal. The `interval_timestamp` setting means that a unique timestamp is applied to all metrics traversing the router during an interval.
 
-**Note**: Use the [message processor configuration](https://github.com/ClusterCockpit/cc-lib/blob/main/messageProcessor/README.md) (option `process_messages`) instead of `add_tags`, `delete_tags`, `drop_metrics`, `drop_metrics_if`, `rename_metrics`, `normalize_units` and `change_unit_prefix`. These options are deprecated and will be removed in future versions. Until then, they are added to the message processor.
+**Note**: Use the [message processor configuration](../../pkg/messageProcessor/README.md) (option `process_messages`) instead of `add_tags`, `delete_tags`, `drop_metrics`, `drop_metrics_if`, `rename_metrics`, `normalize_units` and `change_unit_prefix`. These options are deprecated and will be removed in future versions. Until then, they are added to the message processor.
 
 # Processing order in the router
 
@@ -274,7 +263,7 @@ The above configuration, collects all metric values for metrics evaluating `if`
 If you are not interested in the input metrics `sub_metric_%d+` at all, you can add the same condition used here to the `drop_metrics_if` section to drop them.
 
 Use cases for `interval_aggregates`:
-- Combine multiple metrics of the a collector to a new one like the [MemstatCollector](../../collectors/memstatMetric.md) does it for `mem_used`:
+- Combine multiple metrics of the a collector to a new one like the [MemstatCollector](../../collectors/memstatMetric.md) does it for `mem_used`)):
 ```json
   {
     "name" : "mem_used",

internal/metricRouter/metricCache.go

@@ -4,10 +4,10 @@ import (
 	"sync"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
 	agg "github.com/ClusterCockpit/cc-metric-collector/internal/metricAggregator"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	mct "github.com/ClusterCockpit/cc-metric-collector/pkg/multiChanTicker"
 )
 

internal/metricRouter/metricRouter.go

@@ -8,11 +8,11 @@ import (
 	"sync"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 
-	lp "github.com/ClusterCockpit/cc-lib/ccMessage"
-	mp "github.com/ClusterCockpit/cc-lib/messageProcessor"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
 	agg "github.com/ClusterCockpit/cc-metric-collector/internal/metricAggregator"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
 	mct "github.com/ClusterCockpit/cc-metric-collector/pkg/multiChanTicker"
 )
 
@@ -63,7 +63,7 @@ type metricRouter struct {
 
 // MetricRouter access functions
 type MetricRouter interface {
-	Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfig json.RawMessage) error
+	Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) error
 	AddCollectorInput(input chan lp.CCMessage)
 	AddReceiverInput(input chan lp.CCMessage)
 	AddOutput(output chan lp.CCMessage)
@@ -77,7 +77,7 @@ type MetricRouter interface {
 // * wait group synchronization (from variable wg)
 // * ticker (from variable ticker)
 // * configuration (read from config file in variable routerConfigFile)
-func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfig json.RawMessage) error {
+func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) error {
 	r.outputs = make([]chan lp.CCMessage, 0)
 	r.done = make(chan bool)
 	r.cache_input = make(chan lp.CCMessage)
@@ -95,7 +95,15 @@ func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, rout
 	// Drop domain part of host name
 	r.hostname = strings.SplitN(hostname, `.`, 2)[0]
 
-	err = json.Unmarshal(routerConfig, &r.config)
+	// Read metric router config file
+	configFile, err := os.Open(routerConfigFile)
+	if err != nil {
+		cclog.ComponentError("MetricRouter", err.Error())
+		return err
+	}
+	defer configFile.Close()
+	jsonParser := json.NewDecoder(configFile)
+	err = jsonParser.Decode(&r.config)
 	if err != nil {
 		cclog.ComponentError("MetricRouter", err.Error())
 		return err
@@ -441,9 +449,9 @@ func (r *metricRouter) Close() {
 }
 
 // New creates a new initialized metric router
-func New(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfig json.RawMessage) (MetricRouter, error) {
+func New(ticker mct.MultiChanTicker, wg *sync.WaitGroup, routerConfigFile string) (MetricRouter, error) {
 	r := new(metricRouter)
-	err := r.Init(ticker, wg, routerConfig)
+	err := r.Init(ticker, wg, routerConfigFile)
 	if err != nil {
 		return nil, err
 	}

pkg/ccLogger/cclogger.go

@@ -0,0 +1,113 @@
+package cclogger
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+)
+
+var (
+	globalDebug             = false
+	stdout                  = os.Stdout
+	stderr                  = os.Stderr
+	debugLog    *log.Logger = nil
+	infoLog     *log.Logger = nil
+	errorLog    *log.Logger = nil
+	warnLog     *log.Logger = nil
+	defaultLog  *log.Logger = nil
+)
+
+func initLogger() {
+	if debugLog == nil {
+		debugLog = log.New(stderr, "DEBUG ", log.LstdFlags)
+	}
+	if infoLog == nil {
+		infoLog = log.New(stdout, "INFO ", log.LstdFlags)
+	}
+	if errorLog == nil {
+		errorLog = log.New(stderr, "ERROR ", log.LstdFlags)
+	}
+	if warnLog == nil {
+		warnLog = log.New(stderr, "WARN ", log.LstdFlags)
+	}
+	if defaultLog == nil {
+		defaultLog = log.New(stdout, "", log.LstdFlags)
+	}
+}
+
+func Print(e ...interface{}) {
+	initLogger()
+	defaultLog.Print(e...)
+}
+
+func ComponentPrint(component string, e ...interface{}) {
+	initLogger()
+	defaultLog.Print(fmt.Sprintf("[%s] ", component), e)
+}
+
+func Info(e ...interface{}) {
+	initLogger()
+	infoLog.Print(e...)
+}
+
+func ComponentInfo(component string, e ...interface{}) {
+	initLogger()
+	infoLog.Print(fmt.Sprintf("[%s] ", component), e)
+}
+
+func Debug(e ...interface{}) {
+	initLogger()
+	if globalDebug {
+		debugLog.Print(e...)
+	}
+}
+
+func ComponentDebug(component string, e ...interface{}) {
+	initLogger()
+	if globalDebug && debugLog != nil {
+		//CCComponentPrint(debugLog, component,  e)
+		debugLog.Print(fmt.Sprintf("[%s] ", component), e)
+	}
+}
+
+func Error(e ...interface{}) {
+	initLogger()
+	_, fn, line, _ := runtime.Caller(1)
+	errorLog.Print(fmt.Sprintf("[%s:%d] ", fn, line), e)
+}
+
+func ComponentError(component string, e ...interface{}) {
+	initLogger()
+	_, fn, line, _ := runtime.Caller(1)
+	errorLog.Print(fmt.Sprintf("[%s|%s:%d] ", component, fn, line), e)
+}
+
+func SetDebug() {
+	globalDebug = true
+	initLogger()
+}
+
+func SetOutput(filename string) {
+	if filename == "stderr" {
+		if stderr != os.Stderr && stderr != os.Stdout {
+			stderr.Close()
+		}
+		stderr = os.Stderr
+	} else if filename == "stdout" {
+		if stderr != os.Stderr && stderr != os.Stdout {
+			stderr.Close()
+		}
+		stderr = os.Stdout
+	} else {
+		file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+		if err == nil {
+			defer file.Close()
+			stderr = file
+		}
+	}
+	debugLog = nil
+	errorLog = nil
+	warnLog = nil
+	initLogger()
+}

pkg/ccMetric/README.md

@@ -0,0 +1,57 @@
+# ClusterCockpit metrics
+
+As described in the [ClusterCockpit specifications](https://github.com/ClusterCockpit/cc-specifications), the whole ClusterCockpit stack uses metrics in the InfluxDB line protocol format. This is also the input and output format for the ClusterCockpit Metric Collector but internally it uses an extended format while processing, named CCMetric.
+
+It is basically a copy of the [InfluxDB line protocol](https://github.com/influxdata/line-protocol) `MutableMetric` interface with one extension. Besides the tags and fields, it contains a list of meta information (re-using the `Tag` structure of the original protocol):
+
+```golang
+type ccMetric struct {
+    name   string                 // Measurement name
+    meta   map[string]string      // map of meta data tags
+    tags   map[string]string      // map of of tags
+    fields map[string]interface{} // map of of fields
+    tm     time.Time              // timestamp
+}
+
+type CCMetric interface {
+    ToPoint(metaAsTags map[string]bool) *write.Point  // Generate influxDB point for data type ccMetric
+    ToLineProtocol(metaAsTags map[string]bool) string // Generate influxDB line protocol for data type ccMetric
+    String() string                                   // Return line-protocol like string
+
+    Name() string        // Get metric name
+    SetName(name string) // Set metric name
+
+    Time() time.Time     // Get timestamp
+    SetTime(t time.Time) // Set timestamp
+
+    Tags() map[string]string                   // Map of tags
+    AddTag(key, value string)                  // Add a tag
+    GetTag(key string) (value string, ok bool) // Get a tag by its key
+    HasTag(key string) (ok bool)               // Check if a tag key is present
+    RemoveTag(key string)                      // Remove a tag by its key
+
+    Meta() map[string]string                    // Map of meta data tags
+    AddMeta(key, value string)                  // Add a meta data tag
+    GetMeta(key string) (value string, ok bool) // Get a meta data tab addressed by its key
+    HasMeta(key string) (ok bool)               // Check if a meta data key is present
+    RemoveMeta(key string)                      // Remove a meta data tag by its key
+
+    Fields() map[string]interface{}                   // Map of fields
+    AddField(key string, value interface{})           // Add a field
+    GetField(key string) (value interface{}, ok bool) // Get a field addressed by its key
+    HasField(key string) (ok bool)                    // Check if a field key is present
+    RemoveField(key string)                           // Remove a field addressed by its key
+}
+
+func New(name string, tags map[string]string, meta map[string]string, fields map[string]interface{}, tm time.Time) (CCMetric, error)
+func FromMetric(other CCMetric) CCMetric
+func FromInfluxMetric(other lp.Metric) CCMetric
+```
+
+The `CCMetric` interface provides the same functions as the `MutableMetric` like `{Add, Get, Remove, Has}{Tag, Field}` and additionally provides `{Add, Get, Remove, Has}Meta`.
+
+The InfluxDB protocol creates a new metric with `influx.New(name, tags, fields, time)` while CCMetric uses `ccMetric.New(name, tags, meta, fields, time)` where `tags` and `meta` are both of type `map[string]string`.
+
+You can copy a CCMetric with `FromMetric(other CCMetric) CCMetric`. If you get an `influx.Metric` from a function, like the line protocol parser, you can use `FromInfluxMetric(other influx.Metric) CCMetric` to get a CCMetric out of it (see `NatsReceiver` for an example).
+
+Although the [cc-specifications](https://github.com/ClusterCockpit/cc-specifications/blob/master/interfaces/lineprotocol/README.md) defines that there is only a `value` field for the metric value, the CCMetric still can have multiple values similar to the InfluxDB line protocol.

pkg/ccMetric/ccMetric.go

@@ -0,0 +1,353 @@
+package ccmetric
+
+import (
+	"fmt"
+	"time"
+
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	write "github.com/influxdata/influxdb-client-go/v2/api/write"
+	lp "github.com/influxdata/line-protocol" // MIT license
+	"golang.org/x/exp/maps"
+)
+
+// Most functions are derived from github.com/influxdata/line-protocol/metric.go
+// The metric type is extended with an extra meta information list re-using the Tag
+// type.
+//
+// See: https://docs.influxdata.com/influxdb/latest/reference/syntax/line-protocol/
+type ccMetric struct {
+	name   string                 // Measurement name
+	meta   map[string]string      // map of meta data tags
+	tags   map[string]string      // map of of tags
+	fields map[string]interface{} // map of of fields
+	tm     time.Time              // timestamp
+}
+
+// ccMetric access functions
+type CCMetric interface {
+	ToPoint(metaAsTags map[string]bool) *write.Point  // Generate influxDB point for data type ccMetric
+	ToLineProtocol(metaAsTags map[string]bool) string // Generate influxDB line protocol for data type ccMetric
+
+	Name() string        // Get metric name
+	SetName(name string) // Set metric name
+
+	Time() time.Time     // Get timestamp
+	SetTime(t time.Time) // Set timestamp
+
+	Tags() map[string]string                   // Map of tags
+	AddTag(key, value string)                  // Add a tag
+	GetTag(key string) (value string, ok bool) // Get a tag by its key
+	HasTag(key string) (ok bool)               // Check if a tag key is present
+	RemoveTag(key string)                      // Remove a tag by its key
+
+	Meta() map[string]string                    // Map of meta data tags
+	AddMeta(key, value string)                  // Add a meta data tag
+	GetMeta(key string) (value string, ok bool) // Get a meta data tab addressed by its key
+	HasMeta(key string) (ok bool)               // Check if a meta data key is present
+	RemoveMeta(key string)                      // Remove a meta data tag by its key
+
+	Fields() map[string]interface{}                   // Map of fields
+	AddField(key string, value interface{})           // Add a field
+	GetField(key string) (value interface{}, ok bool) // Get a field addressed by its key
+	HasField(key string) (ok bool)                    // Check if a field key is present
+	RemoveField(key string)                           // Remove a field addressed by its key
+	String() string                                   // Return line-protocol like string
+}
+
+// String implements the stringer interface for data type ccMetric
+func (m *ccMetric) String() string {
+	return fmt.Sprintf(
+		"Name: %s, Tags: %+v, Meta: %+v, fields: %+v, Timestamp: %d",
+		m.name, m.tags, m.meta, m.fields, m.tm.UnixNano(),
+	)
+}
+
+// ToLineProtocol generates influxDB line protocol for data type ccMetric
+func (m *ccMetric) ToPoint(metaAsTags map[string]bool) (p *write.Point) {
+	p = influxdb2.NewPoint(m.name, m.tags, m.fields, m.tm)
+	for key, use_as_tag := range metaAsTags {
+		if use_as_tag {
+			if value, ok := m.GetMeta(key); ok {
+				p.AddTag(key, value)
+			}
+		}
+	}
+	return p
+}
+
+// ToLineProtocol generates influxDB line protocol for data type ccMetric
+func (m *ccMetric) ToLineProtocol(metaAsTags map[string]bool) string {
+
+	return write.PointToLineProtocol(
+		m.ToPoint(metaAsTags),
+		time.Nanosecond,
+	)
+}
+
+// Name returns the measurement name
+func (m *ccMetric) Name() string {
+	return m.name
+}
+
+// SetName sets the measurement name
+func (m *ccMetric) SetName(name string) {
+	m.name = name
+}
+
+// Time returns timestamp
+func (m *ccMetric) Time() time.Time {
+	return m.tm
+}
+
+// SetTime sets the timestamp
+func (m *ccMetric) SetTime(t time.Time) {
+	m.tm = t
+}
+
+// Tags returns the the list of tags as key-value-mapping
+func (m *ccMetric) Tags() map[string]string {
+	return m.tags
+}
+
+// AddTag adds a tag (consisting of key and value) to the map of tags
+func (m *ccMetric) AddTag(key, value string) {
+	m.tags[key] = value
+}
+
+// GetTag returns the tag with tag's key equal to <key>
+func (m *ccMetric) GetTag(key string) (string, bool) {
+	value, ok := m.tags[key]
+	return value, ok
+}
+
+// HasTag checks if a tag with key equal to <key> is present in the list of tags
+func (m *ccMetric) HasTag(key string) bool {
+	_, ok := m.tags[key]
+	return ok
+}
+
+// RemoveTag removes the tag with tag's key equal to <key>
+func (m *ccMetric) RemoveTag(key string) {
+	delete(m.tags, key)
+}
+
+// Meta returns the meta data tags as key-value mapping
+func (m *ccMetric) Meta() map[string]string {
+	return m.meta
+}
+
+// AddMeta adds a meta data tag (consisting of key and value) to the map of meta data tags
+func (m *ccMetric) AddMeta(key, value string) {
+	m.meta[key] = value
+}
+
+// GetMeta returns the meta data tag with meta data's key equal to <key>
+func (m *ccMetric) GetMeta(key string) (string, bool) {
+	value, ok := m.meta[key]
+	return value, ok
+}
+
+// HasMeta checks if a meta data tag with meta data's key equal to <key> is present in the map of meta data tags
+func (m *ccMetric) HasMeta(key string) bool {
+	_, ok := m.meta[key]
+	return ok
+}
+
+// RemoveMeta removes the meta data tag with tag's key equal to <key>
+func (m *ccMetric) RemoveMeta(key string) {
+	delete(m.meta, key)
+}
+
+// Fields returns the list of fields as key-value-mapping
+func (m *ccMetric) Fields() map[string]interface{} {
+	return m.fields
+}
+
+// AddField adds a field (consisting of key and value) to the map of fields
+func (m *ccMetric) AddField(key string, value interface{}) {
+	m.fields[key] = value
+}
+
+// GetField returns the field with field's key equal to <key>
+func (m *ccMetric) GetField(key string) (interface{}, bool) {
+	v, ok := m.fields[key]
+	return v, ok
+}
+
+// HasField checks if a field with field's key equal to <key> is present in the map of fields
+func (m *ccMetric) HasField(key string) bool {
+	_, ok := m.fields[key]
+	return ok
+}
+
+// RemoveField removes the field with field's key equal to <key>
+// from the map of fields
+func (m *ccMetric) RemoveField(key string) {
+	delete(m.fields, key)
+}
+
+// New creates a new measurement point
+func New(
+	name string,
+	tags map[string]string,
+	meta map[string]string,
+	fields map[string]interface{},
+	tm time.Time,
+) (CCMetric, error) {
+	m := &ccMetric{
+		name:   name,
+		tags:   maps.Clone(tags),
+		meta:   maps.Clone(meta),
+		fields: make(map[string]interface{}, len(fields)),
+		tm:     tm,
+	}
+
+	// deep copy fields
+	for k, v := range fields {
+		v := convertField(v)
+		if v == nil {
+			continue
+		}
+		m.fields[k] = v
+	}
+
+	return m, nil
+}
+
+// FromMetric copies the metric <other>
+func FromMetric(other CCMetric) CCMetric {
+
+	return &ccMetric{
+		name:   other.Name(),
+		tags:   maps.Clone(other.Tags()),
+		meta:   maps.Clone(other.Meta()),
+		fields: maps.Clone(other.Fields()),
+		tm:     other.Time(),
+	}
+}
+
+// FromInfluxMetric copies the influxDB line protocol metric <other>
+func FromInfluxMetric(other lp.Metric) CCMetric {
+	m := &ccMetric{
+		name:   other.Name(),
+		tags:   make(map[string]string),
+		meta:   make(map[string]string),
+		fields: make(map[string]interface{}),
+		tm:     other.Time(),
+	}
+
+	// deep copy tags and fields
+	for _, otherTag := range other.TagList() {
+		m.tags[otherTag.Key] = otherTag.Value
+	}
+	for _, otherField := range other.FieldList() {
+		m.fields[otherField.Key] = otherField.Value
+	}
+	return m
+}
+
+// convertField converts data types of fields by the following schemata:
+//
+//	                       *float32, *float64,                      float32, float64 -> float64
+//	*int,  *int8,  *int16,   *int32,   *int64,  int,  int8,  int16,   int32,   int64 ->   int64
+//
+// *uint, *uint8, *uint16,  *uint32,  *uint64, uint, uint8, uint16,  uint32,  uint64 ->  uint64
+// *[]byte, *string,                           []byte, string                        -> string
+// *bool,                                      bool                                  -> bool
+func convertField(v interface{}) interface{} {
+	switch v := v.(type) {
+	case float64:
+		return v
+	case int64:
+		return v
+	case string:
+		return v
+	case bool:
+		return v
+	case int:
+		return int64(v)
+	case uint:
+		return uint64(v)
+	case uint64:
+		return uint64(v)
+	case []byte:
+		return string(v)
+	case int32:
+		return int64(v)
+	case int16:
+		return int64(v)
+	case int8:
+		return int64(v)
+	case uint32:
+		return uint64(v)
+	case uint16:
+		return uint64(v)
+	case uint8:
+		return uint64(v)
+	case float32:
+		return float64(v)
+	case *float64:
+		if v != nil {
+			return *v
+		}
+	case *int64:
+		if v != nil {
+			return *v
+		}
+	case *string:
+		if v != nil {
+			return *v
+		}
+	case *bool:
+		if v != nil {
+			return *v
+		}
+	case *int:
+		if v != nil {
+			return int64(*v)
+		}
+	case *uint:
+		if v != nil {
+			return uint64(*v)
+		}
+	case *uint64:
+		if v != nil {
+			return uint64(*v)
+		}
+	case *[]byte:
+		if v != nil {
+			return string(*v)
+		}
+	case *int32:
+		if v != nil {
+			return int64(*v)
+		}
+	case *int16:
+		if v != nil {
+			return int64(*v)
+		}
+	case *int8:
+		if v != nil {
+			return int64(*v)
+		}
+	case *uint32:
+		if v != nil {
+			return uint64(*v)
+		}
+	case *uint16:
+		if v != nil {
+			return uint64(*v)
+		}
+	case *uint8:
+		if v != nil {
+			return uint64(*v)
+		}
+	case *float32:
+		if v != nil {
+			return float64(*v)
+		}
+	default:
+		return nil
+	}
+	return nil
+}

pkg/ccTopology/ccTopology.go

@@ -9,7 +9,7 @@ import (
 	"strconv"
 	"strings"
 
-	cclogger "github.com/ClusterCockpit/cc-lib/ccLogger"
+	cclogger "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 	"golang.org/x/exp/slices"
 )
 

pkg/messageProcessor/README.md

@@ -0,0 +1,266 @@
+# Message Processor Component
+
+Multiple parts of in the ClusterCockit ecosystem require the processing of CCMessages.
+The main CC application using it is `cc-metric-collector`. The processing part there was originally in the metric router, the central
+hub connecting collectors (reading local data), receivers (receiving remote data) and sinks (sending data). Already in early stages, the
+lack of flexibility caused some trouble:
+
+> The sysadmins wanted to keep operating their Ganglia based monitoring infrastructure while we developed the CC stack. Ganglia wants the core metrics with
+> a specific name and resolution (right unit prefix) but there was no conversion of the data in the CC stack, so CC frontend developers wanted a different
+> resolution for some metrics. The issue was basically the `mem_used` metric showing the currently used memory of the node. Ganglia wants it in `kByte` as provided
+> by the Linux operating system but CC wanted it in `GByte`.
+
+With the message processor, the Ganglia sinks can apply the unit prefix changes individually and name the metrics as required by Ganglia.
+
+## For developers
+
+Whenever you receive or are about to send a message out, you should provide some processing.
+
+### Configuration of component
+
+New operations can be added to the message processor at runtime. Of course, they can also be removed again. For the initial setup, having a configuration file
+or some fields in a configuration file for the processing.
+
+The message processor uses the following configuration
+
+```json
+{
+	"drop_messages": [
+		"name_of_message_to_drop"
+	],
+	"drop_messages_if": [
+		"condition_when_to_drop_message",
+		"name == 'drop_this'",
+		"tag.hostname == 'this_host'",
+		"meta.unit != 'MB'"
+	],
+	"rename_messages" : {
+		"old_message_name" : "new_message_name"
+	},
+	"rename_messages_if": {
+		"condition_when_to_rename_message" : "new_name"
+	},
+	"add_tags_if": [
+		{
+			"if" : "condition_when_to_add_tag",
+			"key": "name_for_new_tag",
+			"value": "new_tag_value"
+		}
+	],
+	"delete_tags_if": [
+		{
+			"if" : "condition_when_to_delete_tag",
+			"key": "name_of_tag"
+		}
+	],
+	"add_meta_if": [
+		{
+			"if" : "condition_when_to_add_meta_info",
+			"key": "name_for_new_meta_info",
+			"value": "new_meta_info_value"
+		}
+	],
+	"delete_meta_if": [
+		{
+			"if" : "condition_when_to_delete_meta_info",
+			"key": "name_of_meta_info"
+		}
+	],
+	"add_field_if": [
+		{
+			"if" : "condition_when_to_add_field",
+			"key": "name_for_new_field",
+			"value": "new_field_value_but_only_string_at_the_moment"
+		}
+	],
+	"delete_field_if": [
+		{
+			"if" : "condition_when_to_delete_field",
+			"key": "name_of_field"
+		}
+	],
+	"move_tag_to_meta_if": [
+		{
+			"if" : "condition_when_to_move_tag_to_meta_info_including_its_value",
+			"key": "name_of_tag",
+			"value": "name_of_meta_info"
+		}
+	],
+	"move_tag_to_field_if": [
+		{
+			"if" : "condition_when_to_move_tag_to_fields_including_its_value",
+			"key": "name_of_tag",
+			"value": "name_of_field"
+		}
+	],
+	"move_meta_to_tag_if": [
+		{
+			"if" : "condition_when_to_move_meta_info_to_tags_including_its_value",
+			"key": "name_of_meta_info",
+			"value": "name_of_tag"
+		}
+	],
+	"move_meta_to_field_if": [
+		{
+			"if" : "condition_when_to_move_meta_info_to_fields_including_its_value",
+			"key": "name_of_tag",
+			"value": "name_of_meta_info"
+		}
+	],
+	"move_field_to_tag_if": [
+		{
+			"if" : "condition_when_to_move_field_to_tags_including_its_stringified_value",
+			"key": "name_of_field",
+			"value": "name_of_tag"
+		}
+	],
+	"move_field_to_meta_if": [
+		{
+			"if" : "condition_when_to_move_field_to_meta_info_including_its_stringified_value",
+			"key": "name_of_field",
+			"value": "name_of_meta_info"
+		}
+	],
+	"drop_by_message_type": [
+		"metric",
+		"event",
+		"log",
+		"control"
+	],
+	"change_unit_prefix": {
+		"name == 'metric_with_wrong_unit_prefix'" : "G",
+		"only_if_messagetype == 'metric'": "T"
+	},
+	"normalize_units": true,
+	"add_base_env": {
+		"MY_CONSTANT_FOR_CUSTOM_CONDITIONS": 1.0,
+		"output_value_for_test_metrics": 42.0,
+	},
+	"stage_order": [
+		"rename_messages_if",
+		"drop_messages"
+	]
+}
+```
+
+The options `change_unit_prefix` and `normalize_units` are only applied to CCMetrics. It is not possible to delete the field related to each message type as defined in [cc-specification](https://github.com/ClusterCockpit/cc-specifications/tree/master/interfaces/lineprotocol). In short:
+- CCMetrics always have to have a field named `value`
+- CCEvents always have to have a field named `event`
+- CCLogs always have to have a field named `log`
+- CCControl messages always have to have a field named `control`
+
+With `add_base_env`, one can specifiy mykey=myvalue pairs that can be used in conditions like `tag.type == mykey`.
+
+The order in which each message is processed, can be specified with the `stage_order` option. The stage names are the keys in the JSON configuration, thus `change_unit_prefix`, `move_field_to_meta_if`, etc. Stages can be listed multiple times.
+
+### Using the component
+In order to load the configuration from a `json.RawMessage`:
+```golang
+mp, err := NewMessageProcessor()
+if err != nil {
+	log.Error("failed to create new message processor")
+}
+mp.FromConfigJSON(configJson)
+```
+
+After initialization and adding the different operations, the `ProcessMessage()` function applies all operations and returns whether the message should be dropped.
+
+```golang
+m := lp.CCMetric{}
+
+x, err := mp.ProcessMessage(m)
+if err != nil {
+	// handle error
+}
+if x != nil {
+    // process x further
+} else {
+	// this message got dropped
+}
+```
+
+Single operations can be added and removed at runtime
+```golang
+type MessageProcessor interface {
+	// Functions to set the execution order of the processing stages
+	SetStages([]string) error
+	DefaultStages() []string
+	// Function to add variables to the base evaluation environment
+	AddBaseEnv(env map[string]interface{}) error
+	// Functions to add and remove rules
+	AddDropMessagesByName(name string) error
+	RemoveDropMessagesByName(name string)
+	AddDropMessagesByCondition(condition string) error
+	RemoveDropMessagesByCondition(condition string)
+	AddRenameMetricByCondition(condition string, name string) error
+	RemoveRenameMetricByCondition(condition string)
+	AddRenameMetricByName(from, to string) error
+	RemoveRenameMetricByName(from string)
+	SetNormalizeUnits(settings bool)
+	AddChangeUnitPrefix(condition string, prefix string) error
+	RemoveChangeUnitPrefix(condition string)
+	AddAddTagsByCondition(condition, key, value string) error
+	RemoveAddTagsByCondition(condition string)
+	AddDeleteTagsByCondition(condition, key, value string) error
+	RemoveDeleteTagsByCondition(condition string)
+	AddAddMetaByCondition(condition, key, value string) error
+	RemoveAddMetaByCondition(condition string)
+	AddDeleteMetaByCondition(condition, key, value string) error
+	RemoveDeleteMetaByCondition(condition string)
+	AddMoveTagToMeta(condition, key, value string) error
+	RemoveMoveTagToMeta(condition string)
+	AddMoveTagToFields(condition, key, value string) error
+	RemoveMoveTagToFields(condition string)
+	AddMoveMetaToTags(condition, key, value string) error
+	RemoveMoveMetaToTags(condition string)
+	AddMoveMetaToFields(condition, key, value string) error
+	RemoveMoveMetaToFields(condition string)
+	AddMoveFieldToTags(condition, key, value string) error
+	RemoveMoveFieldToTags(condition string)
+	AddMoveFieldToMeta(condition, key, value string) error
+	RemoveMoveFieldToMeta(condition string)
+	// Read in a JSON configuration
+	FromConfigJSON(config json.RawMessage) error
+	ProcessMessage(m lp2.CCMessage) (lp2.CCMessage, error)
+	// Processing functions for legacy CCMetric and current CCMessage
+	ProcessMetric(m lp.CCMetric) (lp2.CCMessage, error)
+}
+```
+
+
+### Syntax for evaluatable terms
+
+The message processor uses `gval` for evaluating the terms. It provides a basic set of operators like string comparison and arithmetic operations.
+
+Accessible for operations are
+- `name` of the message
+- `timestamp` or `time` of the message
+- `type`, `type-id` of the message (also `tag_type`, `tag_type-id` and `tag_typeid`)
+- `stype`, `stype-id` of the message (if message has theses tags, also `tag_stype`, `tag_stype-id` and `tag_stypeid`)
+- `value` for a CCMetric message (also `field_value`)
+- `event` for a CCEvent message (also `field_event`)
+- `control` for a CCControl message (also `field_control`)
+- `log` for a CCLog message (also `field_log`)
+- `messagetype` or `msgtype`. Possible values `event`, `metric`, `log` and `control`.
+
+Generally, all tags are accessible with `tag_<tagkey>`, `tags_<tagkey>` or `tags.<tagkey>`. Similarly for all fields with `field[s]?[_.]<fieldkey>`. For meta information `meta[_.]<metakey>` (there is no `metas[_.]<metakey>`).
+
+The [syntax of `expr`](https://expr-lang.org/docs/language-definition) is accepted with some additions:
+- Comparing strings: `==`, `!=`, `str matches regex` (use `%` instead of `\`!)
+- Combining conditions: `&&`, `||`
+- Comparing numbers: `==`, `!=`, `<`, `>`, `<=`, `>=`
+- Test lists: `<value> in <list>`
+- Topological tests: `tag_type-id in getCpuListOfType("socket", "1")` (test if the metric belongs to socket 1 in local node topology)
+
+Often the operations are written in JSON files for loading them at startup. In JSON, some characters are not allowed. Therefore, the term syntax reflects that:
+- use `''` instead of `""` for strings
+- for the regexes, use `%` instead of `\`
+
+
+For operations that should be applied on all messages, use the condition `true`.
+
+### Overhead
+
+The operations taking conditions are pre-processed, which is commonly the time consuming part but, of course, with each added operation, the time to process a message
+increases. Moreover, the processing creates a copy of the message.
+

pkg/messageProcessor/messageProcessor.go

@@ -0,0 +1,988 @@
+package messageprocessor
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"sync"
+
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lplegacy "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
+
+	"github.com/expr-lang/expr"
+	"github.com/expr-lang/expr/vm"
+)
+
+// Message processor add/delete tag/meta configuration
+type messageProcessorTagConfig struct {
+	Key       string `json:"key"`             // Tag name
+	Value     string `json:"value,omitempty"` // Tag value
+	Condition string `json:"if"`              // Condition for adding or removing corresponding tag
+}
+
+type messageProcessorConfig struct {
+	StageOrder       []string                    `json:"stage_order,omitempty"`          // List of stages to execute them in the specified order and to skip unrequired ones
+	DropMessages     []string                    `json:"drop_messages,omitempty"`        // List of metric names to drop. For fine-grained dropping use drop_messages_if
+	DropMessagesIf   []string                    `json:"drop_messages_if,omitempty"`     // List of evaluatable terms to drop messages
+	RenameMessages   map[string]string           `json:"rename_messages,omitempty"`      // Map of metric names to rename
+	RenameMessagesIf map[string]string           `json:"rename_messages_if,omitempty"`   // Map to rename metric name based on a condition
+	NormalizeUnits   bool                        `json:"normalize_units,omitempty"`      // Check unit meta flag and normalize it using cc-units
+	ChangeUnitPrefix map[string]string           `json:"change_unit_prefix,omitempty"`   // Add prefix that should be applied to the messages
+	AddTagsIf        []messageProcessorTagConfig `json:"add_tags_if,omitempty"`          // List of tags that are added when the condition is met
+	DelTagsIf        []messageProcessorTagConfig `json:"delete_tags_if,omitempty"`       // List of tags that are removed when the condition is met
+	AddMetaIf        []messageProcessorTagConfig `json:"add_meta_if,omitempty"`          // List of meta infos that are added when the condition is met
+	DelMetaIf        []messageProcessorTagConfig `json:"delete_meta_if,omitempty"`       // List of meta infos that are removed when the condition is met
+	AddFieldIf       []messageProcessorTagConfig `json:"add_field_if,omitempty"`         // List of fields that are added when the condition is met
+	DelFieldIf       []messageProcessorTagConfig `json:"delete_field_if,omitempty"`      // List of fields that are removed when the condition is met
+	DropByType       []string                    `json:"drop_by_message_type,omitempty"` // List of message types that should be dropped
+	MoveTagToMeta    []messageProcessorTagConfig `json:"move_tag_to_meta_if,omitempty"`
+	MoveTagToField   []messageProcessorTagConfig `json:"move_tag_to_field_if,omitempty"`
+	MoveMetaToTag    []messageProcessorTagConfig `json:"move_meta_to_tag_if,omitempty"`
+	MoveMetaToField  []messageProcessorTagConfig `json:"move_meta_to_field_if,omitempty"`
+	MoveFieldToTag   []messageProcessorTagConfig `json:"move_field_to_tag_if,omitempty"`
+	MoveFieldToMeta  []messageProcessorTagConfig `json:"move_field_to_meta_if,omitempty"`
+	AddBaseEnv       map[string]interface{}      `json:"add_base_env,omitempty"`
+}
+
+type messageProcessor struct {
+
+	// For thread-safety
+	mutex sync.RWMutex
+
+	// mapping contains all evalables as strings to gval.Evaluable
+	// because it is not possible to get the original string out of
+	// a gval.Evaluable
+	mapping map[string]*vm.Program
+
+	stages           []string                 // order of stage execution
+	dropMessages     map[string]struct{}      // internal lookup map
+	dropTypes        map[string]struct{}      // internal lookup map
+	dropMessagesIf   map[*vm.Program]struct{} // pre-processed dropMessagesIf
+	renameMessages   map[string]string        // internal lookup map
+	renameMessagesIf map[*vm.Program]string   // pre-processed RenameMessagesIf
+	changeUnitPrefix map[*vm.Program]string   // pre-processed ChangeUnitPrefix
+	normalizeUnits   bool
+	addTagsIf        map[*vm.Program]messageProcessorTagConfig // pre-processed AddTagsIf
+	deleteTagsIf     map[*vm.Program]messageProcessorTagConfig // pre-processed DelTagsIf
+	addMetaIf        map[*vm.Program]messageProcessorTagConfig // pre-processed AddMetaIf
+	deleteMetaIf     map[*vm.Program]messageProcessorTagConfig // pre-processed DelMetaIf
+	addFieldIf       map[*vm.Program]messageProcessorTagConfig // pre-processed AddFieldIf
+	deleteFieldIf    map[*vm.Program]messageProcessorTagConfig // pre-processed DelFieldIf
+	moveTagToMeta    map[*vm.Program]messageProcessorTagConfig // pre-processed MoveTagToMeta
+	moveTagToField   map[*vm.Program]messageProcessorTagConfig // pre-processed MoveTagToField
+	moveMetaToTag    map[*vm.Program]messageProcessorTagConfig // pre-processed MoveMetaToTag
+	moveMetaToField  map[*vm.Program]messageProcessorTagConfig // pre-processed MoveMetaToField
+	moveFieldToTag   map[*vm.Program]messageProcessorTagConfig // pre-processed MoveFieldToTag
+	moveFieldToMeta  map[*vm.Program]messageProcessorTagConfig // pre-processed MoveFieldToMeta
+}
+
+type MessageProcessor interface {
+	// Functions to set the execution order of the processing stages
+	SetStages([]string) error
+	DefaultStages() []string
+	// Function to add variables to the base evaluation environment
+	AddBaseEnv(env map[string]interface{}) error
+	// Functions to add and remove rules
+	AddDropMessagesByName(name string) error
+	RemoveDropMessagesByName(name string)
+	AddDropMessagesByCondition(condition string) error
+	RemoveDropMessagesByCondition(condition string)
+	AddRenameMetricByCondition(condition string, name string) error
+	RemoveRenameMetricByCondition(condition string)
+	AddRenameMetricByName(from, to string) error
+	RemoveRenameMetricByName(from string)
+	SetNormalizeUnits(settings bool)
+	AddChangeUnitPrefix(condition string, prefix string) error
+	RemoveChangeUnitPrefix(condition string)
+	AddAddTagsByCondition(condition, key, value string) error
+	RemoveAddTagsByCondition(condition string)
+	AddDeleteTagsByCondition(condition, key, value string) error
+	RemoveDeleteTagsByCondition(condition string)
+	AddAddMetaByCondition(condition, key, value string) error
+	RemoveAddMetaByCondition(condition string)
+	AddDeleteMetaByCondition(condition, key, value string) error
+	RemoveDeleteMetaByCondition(condition string)
+	AddMoveTagToMeta(condition, key, value string) error
+	RemoveMoveTagToMeta(condition string)
+	AddMoveTagToFields(condition, key, value string) error
+	RemoveMoveTagToFields(condition string)
+	AddMoveMetaToTags(condition, key, value string) error
+	RemoveMoveMetaToTags(condition string)
+	AddMoveMetaToFields(condition, key, value string) error
+	RemoveMoveMetaToFields(condition string)
+	AddMoveFieldToTags(condition, key, value string) error
+	RemoveMoveFieldToTags(condition string)
+	AddMoveFieldToMeta(condition, key, value string) error
+	RemoveMoveFieldToMeta(condition string)
+	// Read in a JSON configuration
+	FromConfigJSON(config json.RawMessage) error
+	// Processing functions for legacy CCMetric and current CCMessage
+	ProcessMetric(m lplegacy.CCMetric) (lp.CCMessage, error)
+	ProcessMessage(m lp.CCMessage) (lp.CCMessage, error)
+	//EvalToBool(condition string, parameters map[string]interface{}) (bool, error)
+	//EvalToFloat64(condition string, parameters map[string]interface{}) (float64, error)
+	//EvalToString(condition string, parameters map[string]interface{}) (string, error)
+}
+
+const (
+	STAGENAME_DROP_BY_NAME       string = "drop_by_name"
+	STAGENAME_DROP_BY_TYPE       string = "drop_by_type"
+	STAGENAME_DROP_IF            string = "drop_if"
+	STAGENAME_ADD_TAG            string = "add_tag"
+	STAGENAME_DELETE_TAG         string = "delete_tag"
+	STAGENAME_MOVE_TAG_META      string = "move_tag_to_meta"
+	STAGENAME_MOVE_TAG_FIELD     string = "move_tag_to_fields"
+	STAGENAME_ADD_META           string = "add_meta"
+	STAGENAME_DELETE_META        string = "delete_meta"
+	STAGENAME_MOVE_META_TAG      string = "move_meta_to_tags"
+	STAGENAME_MOVE_META_FIELD    string = "move_meta_to_fields"
+	STAGENAME_ADD_FIELD          string = "add_field"
+	STAGENAME_DELETE_FIELD       string = "delete_field"
+	STAGENAME_MOVE_FIELD_TAG     string = "move_field_to_tags"
+	STAGENAME_MOVE_FIELD_META    string = "move_field_to_meta"
+	STAGENAME_RENAME_BY_NAME     string = "rename"
+	STAGENAME_RENAME_IF          string = "rename_if"
+	STAGENAME_CHANGE_UNIT_PREFIX string = "change_unit_prefix"
+	STAGENAME_NORMALIZE_UNIT     string = "normalize_unit"
+)
+
+var StageNames = []string{
+	STAGENAME_DROP_BY_NAME,
+	STAGENAME_DROP_BY_TYPE,
+	STAGENAME_DROP_IF,
+	STAGENAME_ADD_TAG,
+	STAGENAME_DELETE_TAG,
+	STAGENAME_MOVE_TAG_META,
+	STAGENAME_MOVE_TAG_FIELD,
+	STAGENAME_ADD_META,
+	STAGENAME_DELETE_META,
+	STAGENAME_MOVE_META_TAG,
+	STAGENAME_MOVE_META_FIELD,
+	STAGENAME_ADD_FIELD,
+	STAGENAME_DELETE_FIELD,
+	STAGENAME_MOVE_FIELD_TAG,
+	STAGENAME_MOVE_FIELD_META,
+	STAGENAME_RENAME_BY_NAME,
+	STAGENAME_RENAME_IF,
+	STAGENAME_CHANGE_UNIT_PREFIX,
+	STAGENAME_NORMALIZE_UNIT,
+}
+
+var paramMapPool = sync.Pool{
+	New: func() any {
+		return make(map[string]interface{})
+	},
+}
+
+func sanitizeExprString(key string) string {
+	return strings.ReplaceAll(key, "type-id", "typeid")
+}
+
+func getParamMap(point lp.CCMetric) map[string]interface{} {
+	params := paramMapPool.Get().(map[string]interface{})
+	params["message"] = point
+	params["msg"] = point
+	params["name"] = point.Name()
+	params["timestamp"] = point.Time().Unix()
+	params["time"] = params["timestamp"]
+
+	fields := paramMapPool.Get().(map[string]interface{})
+	for key, value := range point.Fields() {
+		fields[key] = value
+		switch key {
+		case "value":
+			params["messagetype"] = "metric"
+			params["value"] = value
+			params["metric"] = value
+		case "event":
+			params["messagetype"] = "event"
+			params["event"] = value
+		case "control":
+			params["messagetype"] = "control"
+			params["control"] = value
+		case "log":
+			params["messagetype"] = "log"
+			params["log"] = value
+		default:
+			params["messagetype"] = "unknown"
+		}
+	}
+	params["msgtype"] = params["messagetype"]
+	params["fields"] = fields
+	params["field"] = fields
+	tags := paramMapPool.Get().(map[string]interface{})
+	for key, value := range point.Tags() {
+		tags[sanitizeExprString(key)] = value
+	}
+	params["tags"] = tags
+	params["tag"] = tags
+	meta := paramMapPool.Get().(map[string]interface{})
+	for key, value := range point.Meta() {
+		meta[sanitizeExprString(key)] = value
+	}
+	params["meta"] = meta
+	return params
+}
+
+var baseenv = map[string]interface{}{
+	"name":        "",
+	"messagetype": "unknown",
+	"msgtype":     "unknown",
+	"tag": map[string]interface{}{
+		"type":     "unknown",
+		"typeid":   "0",
+		"stype":    "unknown",
+		"stypeid":  "0",
+		"hostname": "localhost",
+		"cluster":  "nocluster",
+	},
+	"tags": map[string]interface{}{
+		"type":     "unknown",
+		"typeid":   "0",
+		"stype":    "unknown",
+		"stypeid":  "0",
+		"hostname": "localhost",
+		"cluster":  "nocluster",
+	},
+	"meta": map[string]interface{}{
+		"unit":   "invalid",
+		"source": "unknown",
+	},
+	"fields": map[string]interface{}{
+		"value":   0,
+		"event":   "",
+		"control": "",
+		"log":     "",
+	},
+	"field": map[string]interface{}{
+		"value":   0,
+		"event":   "",
+		"control": "",
+		"log":     "",
+	},
+	"timestamp": 1234567890,
+	"msg":       lp.EmptyMessage(),
+	"message":   lp.EmptyMessage(),
+}
+
+func addBaseEnvWalker(values map[string]interface{}) map[string]interface{} {
+	out := make(map[string]interface{})
+	for k, v := range values {
+		switch value := v.(type) {
+		case int, int32, int64, uint, uint32, uint64, string, float32, float64:
+			out[k] = value
+		case map[string]interface{}:
+			if _, ok := baseenv[k]; !ok {
+				out[k] = addBaseEnvWalker(value)
+			}
+		}
+	}
+	return out
+}
+
+func (mp *messageProcessor) AddBaseEnv(env map[string]interface{}) error {
+	for k, v := range env {
+		switch value := v.(type) {
+		case int, int32, int64, uint, uint32, uint64, string, float32, float64:
+			baseenv[k] = value
+		case map[string]interface{}:
+			if _, ok := baseenv[k]; !ok {
+				baseenv[k] = addBaseEnvWalker(value)
+			}
+		}
+	}
+	return nil
+}
+
+func (mp *messageProcessor) init() error {
+	mp.stages = make([]string, 0)
+	mp.mapping = make(map[string]*vm.Program)
+	mp.dropMessages = make(map[string]struct{})
+	mp.dropTypes = make(map[string]struct{})
+	mp.dropMessagesIf = make(map[*vm.Program]struct{})
+	mp.renameMessages = make(map[string]string)
+	mp.renameMessagesIf = make(map[*vm.Program]string)
+	mp.changeUnitPrefix = make(map[*vm.Program]string)
+	mp.addTagsIf = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.addMetaIf = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.addFieldIf = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.deleteTagsIf = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.deleteMetaIf = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.deleteFieldIf = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.moveFieldToMeta = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.moveFieldToTag = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.moveMetaToField = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.moveMetaToTag = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.moveTagToField = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.moveTagToMeta = make(map[*vm.Program]messageProcessorTagConfig)
+	mp.normalizeUnits = false
+	return nil
+}
+
+func (mp *messageProcessor) AddDropMessagesByName(name string) error {
+	mp.mutex.Lock()
+	if _, ok := mp.dropMessages[name]; !ok {
+		mp.dropMessages[name] = struct{}{}
+	}
+	mp.mutex.Unlock()
+	return nil
+}
+
+func (mp *messageProcessor) RemoveDropMessagesByName(name string) {
+	mp.mutex.Lock()
+	delete(mp.dropMessages, name)
+	mp.mutex.Unlock()
+}
+
+func (mp *messageProcessor) AddDropMessagesByType(typestring string) error {
+	valid := []string{"metric", "event", "control", "log"}
+	isValid := false
+	for _, t := range valid {
+		if t == typestring {
+			isValid = true
+			break
+		}
+	}
+	if isValid {
+		mp.mutex.Lock()
+		if _, ok := mp.dropTypes[typestring]; !ok {
+			cclog.ComponentDebug("MessageProcessor", "Adding type", typestring, "for dropping")
+			mp.dropTypes[typestring] = struct{}{}
+		}
+		mp.mutex.Unlock()
+	} else {
+		return fmt.Errorf("invalid message type %s", typestring)
+	}
+	return nil
+}
+
+func (mp *messageProcessor) RemoveDropMessagesByType(typestring string) {
+	mp.mutex.Lock()
+	delete(mp.dropTypes, typestring)
+	mp.mutex.Unlock()
+}
+
+func (mp *messageProcessor) addTagConfig(condition, key, value string, config *map[*vm.Program]messageProcessorTagConfig) error {
+	var err error
+	evaluable, err := expr.Compile(sanitizeExprString(condition), expr.Env(baseenv), expr.AsBool())
+	if err != nil {
+		return fmt.Errorf("failed to create condition evaluable of '%s': %v", condition, err.Error())
+	}
+	mp.mutex.Lock()
+	if _, ok := (*config)[evaluable]; !ok {
+		mp.mapping[condition] = evaluable
+		(*config)[evaluable] = messageProcessorTagConfig{
+			Condition: condition,
+			Key:       key,
+			Value:     value,
+		}
+	}
+	mp.mutex.Unlock()
+	return nil
+}
+
+func (mp *messageProcessor) removeTagConfig(condition string, config *map[*vm.Program]messageProcessorTagConfig) {
+	mp.mutex.Lock()
+	if e, ok := mp.mapping[condition]; ok {
+		delete(mp.mapping, condition)
+		delete(*config, e)
+	}
+	mp.mutex.Unlock()
+}
+
+func (mp *messageProcessor) AddAddTagsByCondition(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.addTagsIf)
+}
+
+func (mp *messageProcessor) RemoveAddTagsByCondition(condition string) {
+	mp.removeTagConfig(condition, &mp.addTagsIf)
+}
+
+func (mp *messageProcessor) AddDeleteTagsByCondition(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.deleteTagsIf)
+}
+
+func (mp *messageProcessor) RemoveDeleteTagsByCondition(condition string) {
+	mp.removeTagConfig(condition, &mp.deleteTagsIf)
+}
+
+func (mp *messageProcessor) AddAddMetaByCondition(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.addMetaIf)
+}
+
+func (mp *messageProcessor) RemoveAddMetaByCondition(condition string) {
+	mp.removeTagConfig(condition, &mp.addMetaIf)
+}
+
+func (mp *messageProcessor) AddDeleteMetaByCondition(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.deleteMetaIf)
+}
+
+func (mp *messageProcessor) RemoveDeleteMetaByCondition(condition string) {
+	mp.removeTagConfig(condition, &mp.deleteMetaIf)
+}
+
+func (mp *messageProcessor) AddAddFieldByCondition(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.addFieldIf)
+}
+
+func (mp *messageProcessor) RemoveAddFieldByCondition(condition string) {
+	mp.removeTagConfig(condition, &mp.addFieldIf)
+}
+
+func (mp *messageProcessor) AddDeleteFieldByCondition(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.deleteFieldIf)
+}
+
+func (mp *messageProcessor) RemoveDeleteFieldByCondition(condition string) {
+	mp.removeTagConfig(condition, &mp.deleteFieldIf)
+}
+
+func (mp *messageProcessor) AddDropMessagesByCondition(condition string) error {
+
+	var err error
+	evaluable, err := expr.Compile(sanitizeExprString(condition), expr.Env(baseenv), expr.AsBool())
+	if err != nil {
+		return fmt.Errorf("failed to create condition evaluable of '%s': %v", condition, err.Error())
+	}
+	mp.mutex.Lock()
+	if _, ok := mp.dropMessagesIf[evaluable]; !ok {
+		mp.mapping[condition] = evaluable
+		mp.dropMessagesIf[evaluable] = struct{}{}
+	}
+	mp.mutex.Unlock()
+	return nil
+}
+
+func (mp *messageProcessor) RemoveDropMessagesByCondition(condition string) {
+	mp.mutex.Lock()
+	if e, ok := mp.mapping[condition]; ok {
+		delete(mp.mapping, condition)
+		delete(mp.dropMessagesIf, e)
+	}
+	mp.mutex.Unlock()
+}
+
+func (mp *messageProcessor) AddRenameMetricByCondition(condition string, name string) error {
+
+	var err error
+	evaluable, err := expr.Compile(sanitizeExprString(condition), expr.Env(baseenv), expr.AsBool())
+	if err != nil {
+		return fmt.Errorf("failed to create condition evaluable of '%s': %v", condition, err.Error())
+	}
+	mp.mutex.Lock()
+	if _, ok := mp.renameMessagesIf[evaluable]; !ok {
+		mp.mapping[condition] = evaluable
+		mp.renameMessagesIf[evaluable] = name
+	} else {
+		mp.renameMessagesIf[evaluable] = name
+	}
+	mp.mutex.Unlock()
+	return nil
+}
+
+func (mp *messageProcessor) RemoveRenameMetricByCondition(condition string) {
+	mp.mutex.Lock()
+	if e, ok := mp.mapping[condition]; ok {
+		delete(mp.mapping, condition)
+		delete(mp.renameMessagesIf, e)
+	}
+	mp.mutex.Unlock()
+}
+
+func (mp *messageProcessor) SetNormalizeUnits(setting bool) {
+	mp.normalizeUnits = setting
+}
+
+func (mp *messageProcessor) AddChangeUnitPrefix(condition string, prefix string) error {
+
+	var err error
+	evaluable, err := expr.Compile(sanitizeExprString(condition), expr.Env(baseenv), expr.AsBool())
+	if err != nil {
+		return fmt.Errorf("failed to create condition evaluable of '%s': %v", condition, err.Error())
+	}
+	mp.mutex.Lock()
+	if _, ok := mp.changeUnitPrefix[evaluable]; !ok {
+		mp.mapping[condition] = evaluable
+		mp.changeUnitPrefix[evaluable] = prefix
+	} else {
+		mp.changeUnitPrefix[evaluable] = prefix
+	}
+	mp.mutex.Unlock()
+	return nil
+}
+
+func (mp *messageProcessor) RemoveChangeUnitPrefix(condition string) {
+	mp.mutex.Lock()
+	if e, ok := mp.mapping[condition]; ok {
+		delete(mp.mapping, condition)
+		delete(mp.changeUnitPrefix, e)
+	}
+	mp.mutex.Unlock()
+}
+
+func (mp *messageProcessor) AddRenameMetricByName(from, to string) error {
+	mp.mutex.Lock()
+	if _, ok := mp.renameMessages[from]; !ok {
+		mp.renameMessages[from] = to
+	}
+	mp.mutex.Unlock()
+	return nil
+}
+
+func (mp *messageProcessor) RemoveRenameMetricByName(from string) {
+	mp.mutex.Lock()
+	delete(mp.renameMessages, from)
+	mp.mutex.Unlock()
+}
+
+func (mp *messageProcessor) AddMoveTagToMeta(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.moveTagToMeta)
+}
+
+func (mp *messageProcessor) RemoveMoveTagToMeta(condition string) {
+	mp.removeTagConfig(condition, &mp.moveTagToMeta)
+}
+
+func (mp *messageProcessor) AddMoveTagToFields(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.moveTagToField)
+}
+
+func (mp *messageProcessor) RemoveMoveTagToFields(condition string) {
+	mp.removeTagConfig(condition, &mp.moveTagToField)
+}
+
+func (mp *messageProcessor) AddMoveMetaToTags(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.moveMetaToTag)
+}
+
+func (mp *messageProcessor) RemoveMoveMetaToTags(condition string) {
+	mp.removeTagConfig(condition, &mp.moveMetaToTag)
+}
+
+func (mp *messageProcessor) AddMoveMetaToFields(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.moveMetaToField)
+}
+
+func (mp *messageProcessor) RemoveMoveMetaToFields(condition string) {
+	mp.removeTagConfig(condition, &mp.moveMetaToField)
+}
+
+func (mp *messageProcessor) AddMoveFieldToTags(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.moveFieldToTag)
+}
+
+func (mp *messageProcessor) RemoveMoveFieldToTags(condition string) {
+	mp.removeTagConfig(condition, &mp.moveFieldToTag)
+}
+
+func (mp *messageProcessor) AddMoveFieldToMeta(condition, key, value string) error {
+	return mp.addTagConfig(condition, key, value, &mp.moveFieldToMeta)
+}
+
+func (mp *messageProcessor) RemoveMoveFieldToMeta(condition string) {
+	mp.removeTagConfig(condition, &mp.moveFieldToMeta)
+}
+
+func (mp *messageProcessor) SetStages(stages []string) error {
+	newstages := make([]string, 0)
+	if len(stages) == 0 {
+		mp.mutex.Lock()
+		mp.stages = newstages
+		mp.mutex.Unlock()
+		return nil
+	}
+	for i, s := range stages {
+		valid := false
+		for _, v := range StageNames {
+			if s == v {
+				valid = true
+			}
+		}
+		if valid {
+			newstages = append(newstages, s)
+		} else {
+			return fmt.Errorf("invalid stage %s at index %d", s, i)
+		}
+	}
+	mp.mutex.Lock()
+	mp.stages = newstages
+	mp.mutex.Unlock()
+	return nil
+}
+
+func (mp *messageProcessor) DefaultStages() []string {
+	return StageNames
+}
+
+func (mp *messageProcessor) FromConfigJSON(config json.RawMessage) error {
+	var c messageProcessorConfig
+
+	err := json.Unmarshal(config, &c)
+	if err != nil {
+		return fmt.Errorf("failed to process config JSON: %v", err.Error())
+	}
+
+	if len(c.StageOrder) > 0 {
+		err = mp.SetStages(c.StageOrder)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	} else {
+		err = mp.SetStages(mp.DefaultStages())
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+
+	for _, m := range c.DropMessages {
+		err = mp.AddDropMessagesByName(m)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, m := range c.DropByType {
+		err = mp.AddDropMessagesByType(m)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, m := range c.DropMessagesIf {
+		err = mp.AddDropMessagesByCondition(m)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for k, v := range c.RenameMessagesIf {
+		err = mp.AddRenameMetricByCondition(k, v)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for k, v := range c.RenameMessages {
+		err = mp.AddRenameMetricByName(k, v)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for k, v := range c.ChangeUnitPrefix {
+		err = mp.AddChangeUnitPrefix(k, v)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.AddTagsIf {
+		err = mp.AddAddTagsByCondition(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.AddMetaIf {
+		err = mp.AddAddMetaByCondition(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.AddFieldIf {
+		err = mp.AddAddFieldByCondition(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.DelTagsIf {
+		err = mp.AddDeleteTagsByCondition(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.DelMetaIf {
+		err = mp.AddDeleteMetaByCondition(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.DelFieldIf {
+		err = mp.AddDeleteFieldByCondition(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.MoveTagToMeta {
+		err = mp.AddMoveTagToMeta(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.MoveTagToField {
+		err = mp.AddMoveTagToFields(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.MoveMetaToTag {
+		err = mp.AddMoveMetaToTags(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.MoveMetaToField {
+		err = mp.AddMoveMetaToFields(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.MoveFieldToTag {
+		err = mp.AddMoveFieldToTags(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, c := range c.MoveFieldToMeta {
+		err = mp.AddMoveFieldToMeta(c.Condition, c.Key, c.Value)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	for _, m := range c.DropByType {
+		err = mp.AddDropMessagesByType(m)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	if len(c.AddBaseEnv) > 0 {
+		err = mp.AddBaseEnv(c.AddBaseEnv)
+		if err != nil {
+			return fmt.Errorf("failed to process config JSON: %v", err.Error())
+		}
+	}
+	mp.SetNormalizeUnits(c.NormalizeUnits)
+	return nil
+}
+
+func (mp *messageProcessor) ProcessMetric(metric lplegacy.CCMetric) (lp.CCMessage, error) {
+	m, err := lp.NewMessage(
+		metric.Name(),
+		metric.Tags(),
+		metric.Meta(),
+		metric.Fields(),
+		metric.Time(),
+	)
+	if err != nil {
+		return m, fmt.Errorf("failed to parse metric to message: %v", err.Error())
+	}
+	return mp.ProcessMessage(m)
+
+}
+
+func (mp *messageProcessor) ProcessMessage(m lp.CCMessage) (lp.CCMessage, error) {
+	var err error = nil
+	var out lp.CCMessage = lp.FromMessage(m)
+
+	name := out.Name()
+
+	if len(mp.stages) == 0 {
+		mp.SetStages(mp.DefaultStages())
+	}
+
+	mp.mutex.RLock()
+	defer mp.mutex.RUnlock()
+
+	params := getParamMap(out)
+
+	defer func() {
+		params["field"] = nil
+		params["tag"] = nil
+		paramMapPool.Put(params["fields"])
+		paramMapPool.Put(params["tags"])
+		paramMapPool.Put(params["meta"])
+		paramMapPool.Put(params)
+	}()
+
+	for _, s := range mp.stages {
+		switch s {
+		case STAGENAME_DROP_BY_NAME:
+			if len(mp.dropMessages) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Dropping by message name ", name)
+				if _, ok := mp.dropMessages[name]; ok {
+					//cclog.ComponentDebug("MessageProcessor", "Drop")
+					return nil, nil
+				}
+			}
+		case STAGENAME_DROP_BY_TYPE:
+			if len(mp.dropTypes) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Dropping by message type")
+				if _, ok := mp.dropTypes[params["messagetype"].(string)]; ok {
+					//cclog.ComponentDebug("MessageProcessor", "Drop")
+					return nil, nil
+				}
+			}
+		case STAGENAME_DROP_IF:
+			if len(mp.dropMessagesIf) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Dropping by condition")
+				drop, err := dropMessagesIf(&params, &mp.dropMessagesIf)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+				if drop {
+					//cclog.ComponentDebug("MessageProcessor", "Drop")
+					return nil, nil
+				}
+			}
+		case STAGENAME_RENAME_BY_NAME:
+			if len(mp.renameMessages) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Renaming by name match")
+				if newname, ok := mp.renameMessages[name]; ok {
+					//cclog.ComponentDebug("MessageProcessor", "Rename to", newname)
+					out.SetName(newname)
+					//cclog.ComponentDebug("MessageProcessor", "Add old name as 'oldname' to meta", name)
+					out.AddMeta("oldname", name)
+				}
+			}
+		case STAGENAME_RENAME_IF:
+			if len(mp.renameMessagesIf) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Renaming by condition")
+				_, err := renameMessagesIf(out, &params, &mp.renameMessagesIf)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_ADD_TAG:
+			if len(mp.addTagsIf) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Adding tags")
+				_, err = addTagIf(out, &params, &mp.addTagsIf)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_DELETE_TAG:
+			if len(mp.deleteTagsIf) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Delete tags")
+				_, err = deleteTagIf(out, &params, &mp.deleteTagsIf)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_ADD_META:
+			if len(mp.addMetaIf) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Adding meta information")
+				_, err = addMetaIf(out, &params, &mp.addMetaIf)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_DELETE_META:
+			if len(mp.deleteMetaIf) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Delete meta information")
+				_, err = deleteMetaIf(out, &params, &mp.deleteMetaIf)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_ADD_FIELD:
+			if len(mp.addFieldIf) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Adding fields")
+				_, err = addFieldIf(out, &params, &mp.addFieldIf)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_DELETE_FIELD:
+			if len(mp.deleteFieldIf) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Delete fields")
+				_, err = deleteFieldIf(out, &params, &mp.deleteFieldIf)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_MOVE_TAG_META:
+			if len(mp.moveTagToMeta) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Move tag to meta")
+				_, err := moveTagToMeta(out, &params, &mp.moveTagToMeta)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_MOVE_TAG_FIELD:
+			if len(mp.moveTagToField) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Move tag to fields")
+				_, err := moveTagToField(out, &params, &mp.moveTagToField)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_MOVE_META_TAG:
+			if len(mp.moveMetaToTag) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Move meta to tags")
+				_, err := moveMetaToTag(out, &params, &mp.moveMetaToTag)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_MOVE_META_FIELD:
+			if len(mp.moveMetaToField) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Move meta to fields")
+				_, err := moveMetaToField(out, &params, &mp.moveMetaToField)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_MOVE_FIELD_META:
+			if len(mp.moveFieldToMeta) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Move field to meta")
+				_, err := moveFieldToMeta(out, &params, &mp.moveFieldToMeta)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_MOVE_FIELD_TAG:
+			if len(mp.moveFieldToTag) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Move field to tags")
+				_, err := moveFieldToTag(out, &params, &mp.moveFieldToTag)
+				if err != nil {
+					return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+				}
+			}
+		case STAGENAME_NORMALIZE_UNIT:
+			if mp.normalizeUnits {
+				//cclog.ComponentDebug("MessageProcessor", "Normalize units")
+				if lp.IsMetric(out) {
+					_, err := normalizeUnits(out)
+					if err != nil {
+						return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+					}
+				} else {
+					cclog.ComponentDebug("MessageProcessor", "skipped, no metric")
+				}
+			}
+
+		case STAGENAME_CHANGE_UNIT_PREFIX:
+			if len(mp.changeUnitPrefix) > 0 {
+				//cclog.ComponentDebug("MessageProcessor", "Change unit prefix")
+				if lp.IsMetric(out) {
+					_, err := changeUnitPrefix(out, &params, &mp.changeUnitPrefix)
+					if err != nil {
+						return out, fmt.Errorf("failed to evaluate: %v", err.Error())
+					}
+				} else {
+					cclog.ComponentDebug("MessageProcessor", "skipped, no metric")
+				}
+			}
+		}
+
+	}
+
+	return out, nil
+}
+
+// Get a new instace of a message processor.
+func NewMessageProcessor() (MessageProcessor, error) {
+	mp := new(messageProcessor)
+	err := mp.init()
+	if err != nil {
+		err := fmt.Errorf("failed to create MessageProcessor: %v", err.Error())
+		cclog.ComponentError("MessageProcessor", err.Error())
+		return nil, err
+	}
+	return mp, nil
+}

pkg/messageProcessor/messageProcessorFuncs.go

@@ -0,0 +1,262 @@
+package messageprocessor
+
+import (
+	"errors"
+	"fmt"
+
+	lp2 "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	units "github.com/ClusterCockpit/cc-units"
+	"github.com/expr-lang/expr"
+	"github.com/expr-lang/expr/vm"
+)
+
+type MessageLocation int
+
+const (
+	MESSAGE_LOCATION_TAGS MessageLocation = iota
+	MESSAGE_LOCATION_META
+	MESSAGE_LOCATION_FIELDS
+)
+
+// Abstract function to move entries from one location to another
+func moveInMessage(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig, from, to MessageLocation) (bool, error) {
+	for d, data := range *checks {
+		value, err := expr.Run(d, *params)
+		if err != nil {
+			return false, fmt.Errorf("failed to evaluate: %v", err.Error())
+		}
+		//cclog.ComponentDebug("MessageProcessor", "Move from", from, "to", to)
+		if value.(bool) {
+			var v string
+			var ok bool = false
+			switch from {
+			case MESSAGE_LOCATION_TAGS:
+				//cclog.ComponentDebug("MessageProcessor", "Getting tag key", data.Key)
+				v, ok = message.GetTag(data.Key)
+			case MESSAGE_LOCATION_META:
+				//cclog.ComponentDebug("MessageProcessor", "Getting meta key", data.Key)
+				//cclog.ComponentDebug("MessageProcessor", message.Meta())
+				v, ok = message.GetMeta(data.Key)
+			case MESSAGE_LOCATION_FIELDS:
+				var x interface{}
+				//cclog.ComponentDebug("MessageProcessor", "Getting field key", data.Key)
+				x, ok = message.GetField(data.Key)
+				v = fmt.Sprintf("%v", x)
+			}
+			if ok {
+				switch from {
+				case MESSAGE_LOCATION_TAGS:
+					//cclog.ComponentDebug("MessageProcessor", "Removing tag key", data.Key)
+					message.RemoveTag(data.Key)
+				case MESSAGE_LOCATION_META:
+					//cclog.ComponentDebug("MessageProcessor", "Removing meta key", data.Key)
+					message.RemoveMeta(data.Key)
+				case MESSAGE_LOCATION_FIELDS:
+					//cclog.ComponentDebug("MessageProcessor", "Removing field key", data.Key)
+					message.RemoveField(data.Key)
+				}
+				switch to {
+				case MESSAGE_LOCATION_TAGS:
+					//cclog.ComponentDebug("MessageProcessor", "Adding tag", data.Value, "->", v)
+					message.AddTag(data.Value, v)
+				case MESSAGE_LOCATION_META:
+					//cclog.ComponentDebug("MessageProcessor", "Adding meta", data.Value, "->", v)
+					message.AddMeta(data.Value, v)
+				case MESSAGE_LOCATION_FIELDS:
+					//cclog.ComponentDebug("MessageProcessor", "Adding field", data.Value, "->", v)
+					message.AddField(data.Value, v)
+				}
+			}
+		}
+	}
+	return false, nil
+}
+
+func deleteIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig, location MessageLocation) (bool, error) {
+	for d, data := range *checks {
+		value, err := expr.Run(d, *params)
+		if err != nil {
+			return true, fmt.Errorf("failed to evaluate: %v", err.Error())
+		}
+		if value.(bool) {
+			switch location {
+			case MESSAGE_LOCATION_FIELDS:
+				switch data.Key {
+				case "value", "event", "log", "control":
+					return false, errors.New("cannot delete protected fields")
+				default:
+					//cclog.ComponentDebug("MessageProcessor", "Removing field for", data.Key)
+					message.RemoveField(data.Key)
+				}
+			case MESSAGE_LOCATION_TAGS:
+				//cclog.ComponentDebug("MessageProcessor", "Removing tag for", data.Key)
+				message.RemoveTag(data.Key)
+			case MESSAGE_LOCATION_META:
+				//cclog.ComponentDebug("MessageProcessor", "Removing meta for", data.Key)
+				message.RemoveMeta(data.Key)
+			}
+		}
+	}
+	return false, nil
+}
+
+func addIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig, location MessageLocation) (bool, error) {
+	for d, data := range *checks {
+		value, err := expr.Run(d, *params)
+		if err != nil {
+			return true, fmt.Errorf("failed to evaluate: %v", err.Error())
+		}
+		if value.(bool) {
+			switch location {
+			case MESSAGE_LOCATION_FIELDS:
+				//cclog.ComponentDebug("MessageProcessor", "Adding field", data.Value, "->", data.Value)
+				message.AddField(data.Key, data.Value)
+			case MESSAGE_LOCATION_TAGS:
+				//cclog.ComponentDebug("MessageProcessor", "Adding tag", data.Value, "->", data.Value)
+				message.AddTag(data.Key, data.Value)
+			case MESSAGE_LOCATION_META:
+				//cclog.ComponentDebug("MessageProcessor", "Adding meta", data.Value, "->", data.Value)
+				message.AddMeta(data.Key, data.Value)
+			}
+		}
+	}
+	return false, nil
+}
+
+func deleteTagIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return deleteIf(message, params, checks, MESSAGE_LOCATION_TAGS)
+}
+
+func addTagIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return addIf(message, params, checks, MESSAGE_LOCATION_TAGS)
+}
+
+func moveTagToMeta(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return moveInMessage(message, params, checks, MESSAGE_LOCATION_TAGS, MESSAGE_LOCATION_META)
+}
+
+func moveTagToField(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return moveInMessage(message, params, checks, MESSAGE_LOCATION_TAGS, MESSAGE_LOCATION_FIELDS)
+}
+
+func deleteMetaIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return deleteIf(message, params, checks, MESSAGE_LOCATION_META)
+}
+
+func addMetaIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return addIf(message, params, checks, MESSAGE_LOCATION_META)
+}
+
+func moveMetaToTag(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return moveInMessage(message, params, checks, MESSAGE_LOCATION_META, MESSAGE_LOCATION_TAGS)
+}
+
+func moveMetaToField(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return moveInMessage(message, params, checks, MESSAGE_LOCATION_META, MESSAGE_LOCATION_FIELDS)
+}
+
+func deleteFieldIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return deleteIf(message, params, checks, MESSAGE_LOCATION_FIELDS)
+}
+
+func addFieldIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return addIf(message, params, checks, MESSAGE_LOCATION_FIELDS)
+}
+
+func moveFieldToTag(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return moveInMessage(message, params, checks, MESSAGE_LOCATION_FIELDS, MESSAGE_LOCATION_TAGS)
+}
+
+func moveFieldToMeta(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]messageProcessorTagConfig) (bool, error) {
+	return moveInMessage(message, params, checks, MESSAGE_LOCATION_FIELDS, MESSAGE_LOCATION_META)
+}
+
+func dropMessagesIf(params *map[string]interface{}, checks *map[*vm.Program]struct{}) (bool, error) {
+	for d := range *checks {
+		value, err := expr.Run(d, *params)
+		if err != nil {
+			return false, fmt.Errorf("failed to evaluate: %v", err.Error())
+		}
+		if value.(bool) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func normalizeUnits(message lp2.CCMessage) (bool, error) {
+	if in_unit, ok := message.GetMeta("unit"); ok {
+		u := units.NewUnit(in_unit)
+		if u.Valid() {
+			//cclog.ComponentDebug("MessageProcessor", "Update unit with", u.Short())
+			message.AddMeta("unit", u.Short())
+		}
+	} else if in_unit, ok := message.GetTag("unit"); ok {
+		u := units.NewUnit(in_unit)
+		if u.Valid() {
+			//cclog.ComponentDebug("MessageProcessor", "Update unit with", u.Short())
+			message.AddTag("unit", u.Short())
+		}
+	}
+	return false, nil
+}
+
+func changeUnitPrefix(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]string) (bool, error) {
+	for r, n := range *checks {
+		value, err := expr.Run(r, *params)
+		if err != nil {
+			return false, fmt.Errorf("failed to evaluate: %v", err.Error())
+		}
+		if value.(bool) {
+			newPrefix := units.NewPrefix(n)
+			//cclog.ComponentDebug("MessageProcessor", "Condition matches, change to prefix", newPrefix.String())
+			if in_unit, ok := message.GetMeta("unit"); ok && newPrefix != units.InvalidPrefix {
+				u := units.NewUnit(in_unit)
+				if u.Valid() {
+					//cclog.ComponentDebug("MessageProcessor", "Input unit", u.Short())
+					conv, out_unit := units.GetUnitPrefixFactor(u, newPrefix)
+					if conv != nil && out_unit.Valid() {
+						if val, ok := message.GetField("value"); ok {
+							//cclog.ComponentDebug("MessageProcessor", "Update unit with", out_unit.Short())
+							message.AddField("value", conv(val))
+							message.AddMeta("unit", out_unit.Short())
+						}
+					}
+				}
+
+			} else if in_unit, ok := message.GetTag("unit"); ok && newPrefix != units.InvalidPrefix {
+				u := units.NewUnit(in_unit)
+				if u.Valid() {
+					//cclog.ComponentDebug("MessageProcessor", "Input unit", u.Short())
+					conv, out_unit := units.GetUnitPrefixFactor(u, newPrefix)
+					if conv != nil && out_unit.Valid() {
+						if val, ok := message.GetField("value"); ok {
+							//cclog.ComponentDebug("MessageProcessor", "Update unit with", out_unit.Short())
+							message.AddField("value", conv(val))
+							message.AddTag("unit", out_unit.Short())
+						}
+					}
+				}
+
+			}
+		}
+	}
+	return false, nil
+}
+
+func renameMessagesIf(message lp2.CCMessage, params *map[string]interface{}, checks *map[*vm.Program]string) (bool, error) {
+	for d, n := range *checks {
+		value, err := expr.Run(d, *params)
+		if err != nil {
+			return true, fmt.Errorf("failed to evaluate: %v", err.Error())
+		}
+		if value.(bool) {
+			old := message.Name()
+			//cclog.ComponentDebug("MessageProcessor", "Rename to", n)
+			message.SetName(n)
+			//cclog.ComponentDebug("MessageProcessor", "Add old name as 'oldname' to meta", old)
+			message.AddMeta("oldname", old)
+		}
+	}
+	return false, nil
+}

pkg/messageProcessor/messageProcessor_test.go

@@ -0,0 +1,396 @@
+package messageprocessor
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+)
+
+func generate_message_lists(num_lists, num_entries int) ([][]lp.CCMessage, error) {
+	mlist := make([][]lp.CCMessage, 0)
+	for j := 0; j < num_lists; j++ {
+		out := make([]lp.CCMessage, 0)
+		for i := 0; i < num_entries; i++ {
+			var x lp.CCMessage
+			var err error = nil
+			switch {
+			case i%4 == 0:
+				x, err = lp.NewEvent("myevent", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{}, "nothing happend", time.Now())
+			case i%4 == 1:
+				x, err = lp.NewMetric("mymetric", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{"unit": "kByte"}, 12.145, time.Now())
+			case i%4 == 2:
+				x, err = lp.NewLog("mylog", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{}, "disk status: OK", time.Now())
+			case i%4 == 3:
+				x, err = lp.NewGetControl("mycontrol", map[string]string{"type": "socket", "type-id": "0"}, map[string]string{}, time.Now())
+			}
+			if err == nil {
+				x.AddTag("hostname", "myhost")
+				out = append(out, x)
+			} else {
+				return nil, errors.New("failed to create message")
+			}
+		}
+		mlist = append(mlist, out)
+	}
+	return mlist, nil
+}
+
+func TestNewMessageProcessor(t *testing.T) {
+	_, err := NewMessageProcessor()
+	if err != nil {
+		t.Error(err.Error())
+	}
+}
+
+type Configs struct {
+	name   string
+	config json.RawMessage
+	drop   bool
+	errors bool
+	pre    func(msg lp.CCMessage) error
+	check  func(msg lp.CCMessage) error
+}
+
+var test_configs = []Configs{
+	{
+		name:   "single_dropif_nomatch",
+		config: json.RawMessage(`{"drop_messages_if": [ "name == 'testname' && tags.type == 'socket' && tags.typeid % 2 == 1"]}`),
+	},
+	{
+		name:   "drop_by_name",
+		config: json.RawMessage(`{"drop_messages": [ "net_bytes_in"]}`),
+		drop:   true,
+	},
+	{
+		name:   "drop_by_type_match",
+		config: json.RawMessage(`{"drop_by_message_type": [ "metric"]}`),
+		drop:   true,
+	},
+	{
+		name:   "drop_by_type_nomatch",
+		config: json.RawMessage(`{"drop_by_message_type": [ "event"]}`),
+	},
+	{
+		name:   "single_dropif_match",
+		config: json.RawMessage(`{"drop_messages_if": [ "name == 'net_bytes_in' && tags.type == 'node'"]}`),
+		drop:   true,
+	},
+	{
+		name:   "double_dropif_match_nomatch",
+		config: json.RawMessage(`{"drop_messages_if": [ "name == 'net_bytes_in' && tags.type == 'node'", "name == 'testname' && tags.type == 'socket' && tags.typeid % 2 == 1"]}`),
+		drop:   true,
+	},
+	{
+		name:   "rename_simple",
+		config: json.RawMessage(`{"rename_messages": { "net_bytes_in" : "net_bytes_out", "rapl_power": "cpu_power"}}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.Name() != "net_bytes_out" {
+				return errors.New("expected name net_bytes_out but still have net_bytes_in")
+			}
+			return nil
+		},
+	},
+	{
+		name:   "rename_match",
+		config: json.RawMessage(`{"rename_messages_if": { "name == 'net_bytes_in'" : "net_bytes_out", "name == 'rapl_power'": "cpu_power"}}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.Name() != "net_bytes_out" {
+				return errors.New("expected name net_bytes_out but still have net_bytes_in")
+			}
+			return nil
+		},
+	},
+	{
+		name:   "rename_nomatch",
+		config: json.RawMessage(`{"rename_messages_if": { "name == 'net_bytes_out'" : "net_bytes_in", "name == 'rapl_power'": "cpu_power"}}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.Name() != "net_bytes_in" {
+				return errors.New("expected name net_bytes_in but still have net_bytes_out")
+			}
+			return nil
+		},
+	},
+	{
+		name:   "add_tag",
+		config: json.RawMessage(`{"add_tags_if": [{"if": "name == 'net_bytes_in'", "key" : "cluster", "value" : "mycluster"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if !msg.HasTag("cluster") {
+				return errors.New("expected new tag 'cluster' but not present")
+			}
+			return nil
+		},
+	},
+	{
+		name:   "del_tag",
+		config: json.RawMessage(`{"delete_tags_if": [{"if": "name == 'net_bytes_in'", "key" : "type"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.HasTag("type") {
+				return errors.New("expected to have no 'type' but still present")
+			}
+			return nil
+		},
+	},
+	{
+		name:   "add_meta",
+		config: json.RawMessage(`{"add_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "source", "value" : "example"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if !msg.HasMeta("source") {
+				return errors.New("expected new tag 'source' but not present")
+			}
+			return nil
+		},
+	},
+	{
+		name:   "del_meta",
+		config: json.RawMessage(`{"delete_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "unit"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.HasMeta("unit") {
+				return errors.New("expected to have no 'unit' but still present")
+			}
+			return nil
+		},
+	},
+	{
+		name:   "add_field",
+		config: json.RawMessage(`{"add_fields_if": [{"if": "name == 'net_bytes_in'", "key" : "myfield", "value" : "example"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if !msg.HasField("myfield") {
+				return errors.New("expected new tag 'source' but not present")
+			}
+			return nil
+		},
+	},
+	{
+		name:   "delete_fields_if_protected",
+		config: json.RawMessage(`{"delete_fields_if": [{"if": "name == 'net_bytes_in'", "key" : "value"}]}`),
+		errors: true,
+		check: func(msg lp.CCMessage) error {
+			if !msg.HasField("value") {
+				return errors.New("expected to still have 'value' field because it is a protected field key")
+			}
+			return nil
+		},
+	},
+	{
+		name:   "delete_fields_if_unprotected",
+		config: json.RawMessage(`{"delete_fields_if": [{"if": "name == 'net_bytes_in'", "key" : "testfield"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.HasField("testfield") {
+				return errors.New("expected to still have 'testfield' field but should be deleted")
+			}
+			return nil
+		},
+		pre: func(msg lp.CCMessage) error {
+			msg.AddField("testfield", 4.123)
+			return nil
+		},
+	},
+	{
+		name:   "single_change_prefix_match",
+		config: json.RawMessage(`{"change_unit_prefix": {"name == 'net_bytes_in' && tags.type == 'node'": "M"}}`),
+		check: func(msg lp.CCMessage) error {
+			if u, ok := msg.GetMeta("unit"); ok {
+				if u != "MB" {
+					return fmt.Errorf("expected unit MB but have %s", u)
+				}
+			} else if u, ok := msg.GetTag("unit"); ok {
+				if u != "MB" {
+					return fmt.Errorf("expected unit MB but have %s", u)
+				}
+			}
+			return nil
+		},
+	},
+	{
+		name:   "normalize_units",
+		config: json.RawMessage(`{"normalize_units": true}`),
+		check: func(msg lp.CCMessage) error {
+			if u, ok := msg.GetMeta("unit"); ok {
+				if u != "B" {
+					return fmt.Errorf("expected unit B but have %s", u)
+				}
+			} else if u, ok := msg.GetTag("unit"); ok {
+				if u != "B" {
+					return fmt.Errorf("expected unit B but have %s", u)
+				}
+			}
+			return nil
+		},
+	},
+	{
+		name:   "move_tag_to_meta",
+		config: json.RawMessage(`{"move_tag_to_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "type-id", "value": "typeid"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.HasTag("type-id") || !msg.HasMeta("typeid") {
+				return errors.New("moving tag 'type-id' to meta 'typeid' failed")
+			}
+			return nil
+		},
+		pre: func(msg lp.CCMessage) error {
+			msg.AddTag("type-id", "0")
+			return nil
+		},
+	},
+	{
+		name:   "move_tag_to_field",
+		config: json.RawMessage(`{"move_tag_to_field_if": [{"if": "name == 'net_bytes_in'", "key" : "type-id", "value": "typeid"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.HasTag("type-id") || !msg.HasField("typeid") {
+				return errors.New("moving tag 'type-id' to field 'typeid' failed")
+			}
+			return nil
+		},
+		pre: func(msg lp.CCMessage) error {
+			msg.AddTag("type-id", "0")
+			return nil
+		},
+	},
+	{
+		name:   "move_meta_to_tag",
+		config: json.RawMessage(`{"move_meta_to_tag_if": [{"if": "name == 'net_bytes_in'", "key" : "unit", "value": "unit"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.HasMeta("unit") || !msg.HasTag("unit") {
+				return errors.New("moving meta 'unit' to tag 'unit' failed")
+			}
+			return nil
+		},
+	},
+	{
+		name:   "move_meta_to_field",
+		config: json.RawMessage(`{"move_meta_to_field_if": [{"if": "name == 'net_bytes_in'", "key" : "unit", "value": "unit"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.HasMeta("unit") || !msg.HasField("unit") {
+				return errors.New("moving meta 'unit' to field 'unit' failed")
+			}
+			return nil
+		},
+	},
+	{
+		name:   "move_field_to_tag",
+		config: json.RawMessage(`{"move_field_to_tag_if": [{"if": "name == 'net_bytes_in'", "key" : "myfield", "value": "field"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.HasField("myfield") || !msg.HasTag("field") {
+				return errors.New("moving meta 'myfield' to tag 'field' failed")
+			}
+			return nil
+		},
+		pre: func(msg lp.CCMessage) error {
+			msg.AddField("myfield", 12)
+			return nil
+		},
+	},
+	{
+		name:   "move_field_to_meta",
+		config: json.RawMessage(`{"move_field_to_meta_if": [{"if": "name == 'net_bytes_in'", "key" : "myfield", "value": "field"}]}`),
+		check: func(msg lp.CCMessage) error {
+			if msg.HasField("myfield") || !msg.HasMeta("field") {
+				return errors.New("moving meta 'myfield' to meta 'field' failed")
+			}
+			return nil
+		},
+		pre: func(msg lp.CCMessage) error {
+			msg.AddField("myfield", 12)
+			return nil
+		},
+	},
+}
+
+func TestConfigList(t *testing.T) {
+	for _, c := range test_configs {
+		t.Run(c.name, func(t *testing.T) {
+			m, err := lp.NewMetric("net_bytes_in", map[string]string{"type": "node", "type-id": "0"}, map[string]string{"unit": "Byte"}, float64(1024.0), time.Now())
+			if err != nil {
+				t.Error(err.Error())
+				return
+			}
+			if c.pre != nil {
+				if err = c.pre(m); err != nil {
+					t.Errorf("error running pre-test function: %v", err.Error())
+					return
+				}
+			}
+
+			mp, err := NewMessageProcessor()
+			if err != nil {
+				t.Error(err.Error())
+				return
+			}
+			err = mp.FromConfigJSON(c.config)
+			if err != nil {
+				t.Error(err.Error())
+				return
+			}
+			//t.Log(m.ToLineProtocol(nil))
+			out, err := mp.ProcessMessage(m)
+			if err != nil && !c.errors {
+				cclog.SetDebug()
+				mp.ProcessMessage(m)
+				t.Error(err.Error())
+				return
+			}
+			if out == nil && !c.drop {
+				t.Error("fail, message should NOT be dropped but processor signalled dropping")
+				return
+			} else if out != nil && c.drop {
+				t.Error("fail, message should be dropped but processor signalled NO dropping")
+				return
+			}
+			// {
+			// 	if c.drop {
+			// 		t.Error("fail, message should be dropped but processor signalled NO dropping")
+			// 	} else {
+			// 		t.Error("fail, message should NOT be dropped but processor signalled dropping")
+			// 	}
+			// 	cclog.SetDebug()
+			// 	mp.ProcessMessage(m)
+			// 	return
+			// }
+			if c.check != nil {
+				if err := c.check(out); err != nil {
+					t.Errorf("check failed with %v", err.Error())
+					t.Log("Rerun with debugging")
+					cclog.SetDebug()
+					mp.ProcessMessage(m)
+					return
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkProcessing(b *testing.B) {
+
+	mlist, err := generate_message_lists(b.N, 1000)
+	if err != nil {
+		b.Error(err.Error())
+		return
+	}
+
+	mp, err := NewMessageProcessor()
+	if err != nil {
+		b.Error(err.Error())
+		return
+	}
+	err = mp.FromConfigJSON(json.RawMessage(`{"move_meta_to_tag_if": [{"if" : "name == 'mymetric'", "key":"unit", "value":"unit"}]}`))
+	if err != nil {
+		b.Error(err.Error())
+		return
+	}
+
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		for _, m := range mlist[i] {
+			if _, err := mp.ProcessMessage(m); err != nil {
+				b.Errorf("failed processing message '%s': %v", m.ToLineProtocol(nil), err.Error())
+				return
+			}
+		}
+	}
+	b.StopTimer()
+	b.ReportMetric(float64(b.Elapsed())/float64(len(mlist)*b.N), "ns/message")
+}

pkg/multiChanTicker/README.md

@@ -1,14 +1,3 @@
-<!--
----
-title: Multi-channel Ticker
-description: Timer ticker that sends out the tick to multiple channels
-categories: [cc-metric-collector]
-tags: ['Developer']
-weight: 1
-hugo_path: docs/reference/cc-metric-collector/pkg/multichanticker/_index.md
----
--->
-
 # MultiChanTicker
 
 The idea of this ticker is to multiply the output channels. The original Golang `time.Ticker` provides only a single output channel, so the signal can only be received by a single other class. This ticker allows to add multiple channels which get all notified about the time tick.

pkg/multiChanTicker/multiChanTicker.go

@@ -3,7 +3,7 @@ package multiChanTicker
 import (
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
 )
 
 type multiChanTicker struct {

receivers/README.md

@@ -0,0 +1,32 @@
+# CCMetric receivers
+
+This folder contains the ReceiveManager and receiver implementations for the cc-metric-collector.
+
+## Configuration
+
+The configuration file for the receivers is a list of configurations. The `type` field in each specifies which receiver to initialize.
+
+```json
+{
+  "myreceivername" : {
+    "type": "receiver-type",
+    <receiver-specific configuration>
+  }
+}
+```
+
+This allows to specify
+
+## Available receivers
+
+- [`nats`](./natsReceiver.md): Receive metrics from the NATS network
+- [`prometheus`](./prometheusReceiver.md): Scrape data from a Prometheus client
+- [`http`](./httpReceiver.md): Listen for HTTP Post requests transporting metrics in InfluxDB line protocol
+- [`ipmi`](./ipmiReceiver.md): Read IPMI sensor readings
+- [`redfish`](redfishReceiver.md) Use the Redfish (specification) to query thermal and power metrics
+
+## Contributing own receivers
+
+A receiver contains a few functions and is derived from the type `Receiver` (in `metricReceiver.go`):
+
+For an example, check the [sample receiver](./sampleReceiver.go)

receivers/httpReceiver.go

@@ -0,0 +1,183 @@
+package receivers
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
+)
+
+const HTTP_RECEIVER_PORT = "8080"
+
+type HttpReceiverConfig struct {
+	defaultReceiverConfig
+	Addr string `json:"address"`
+	Port string `json:"port"`
+	Path string `json:"path"`
+
+	// Maximum amount of time to wait for the next request when keep-alives are enabled
+	// should be larger than the measurement interval to keep the connection open
+	IdleTimeout string `json:"idle_timeout"`
+	idleTimeout time.Duration
+
+	// Controls whether HTTP keep-alives are enabled. By default, keep-alives are enabled
+	KeepAlivesEnabled bool `json:"keep_alives_enabled"`
+
+	// Basic authentication
+	Username     string `json:"username"`
+	Password     string `json:"password"`
+	useBasicAuth bool
+}
+
+type HttpReceiver struct {
+	receiver
+	//meta   map[string]string
+	config HttpReceiverConfig
+	server *http.Server
+	wg     sync.WaitGroup
+}
+
+func (r *HttpReceiver) Init(name string, config json.RawMessage) error {
+	r.name = fmt.Sprintf("HttpReceiver(%s)", name)
+
+	// Set default values
+	r.config.Port = HTTP_RECEIVER_PORT
+	r.config.KeepAlivesEnabled = true
+	// should be larger than the measurement interval to keep the connection open
+	r.config.IdleTimeout = "120s"
+
+	// Read config
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &r.config)
+		if err != nil {
+			cclog.ComponentError(r.name, "Error reading config:", err.Error())
+			return err
+		}
+	}
+	if len(r.config.Port) == 0 {
+		return errors.New("not all configuration variables set required by HttpReceiver")
+	}
+
+	// Check idle timeout config
+	if len(r.config.IdleTimeout) > 0 {
+		t, err := time.ParseDuration(r.config.IdleTimeout)
+		if err == nil {
+			cclog.ComponentDebug(r.name, "idleTimeout", t)
+			r.config.idleTimeout = t
+		}
+	}
+
+	// Check basic authentication config
+	if len(r.config.Username) > 0 || len(r.config.Password) > 0 {
+		r.config.useBasicAuth = true
+	}
+	if r.config.useBasicAuth && len(r.config.Username) == 0 {
+		return errors.New("basic authentication requires username")
+	}
+	if r.config.useBasicAuth && len(r.config.Password) == 0 {
+		return errors.New("basic authentication requires password")
+	}
+	msgp, err := mp.NewMessageProcessor()
+	if err != nil {
+		return fmt.Errorf("initialization of message processor failed: %v", err.Error())
+	}
+	r.mp = msgp
+	if len(r.config.MessageProcessor) > 0 {
+		err = r.mp.FromConfigJSON(r.config.MessageProcessor)
+		if err != nil {
+			return fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+		}
+	}
+	r.mp.AddAddMetaByCondition("true", "source", r.name)
+
+	//r.meta = map[string]string{"source": r.name}
+	p := r.config.Path
+	if !strings.HasPrefix(p, "/") {
+		p = "/" + p
+	}
+	addr := fmt.Sprintf("%s:%s", r.config.Addr, r.config.Port)
+	uri := addr + p
+	cclog.ComponentDebug(r.name, "INIT", "listen on:", uri)
+
+	// Register handler function r.ServerHttp for path p in the DefaultServeMux
+	http.HandleFunc(p, r.ServerHttp)
+
+	// Create http server
+	r.server = &http.Server{
+		Addr:        addr,
+		Handler:     nil, // handler to invoke, http.DefaultServeMux if nil
+		IdleTimeout: r.config.idleTimeout,
+	}
+	r.server.SetKeepAlivesEnabled(r.config.KeepAlivesEnabled)
+
+	return nil
+}
+
+func (r *HttpReceiver) Start() {
+	cclog.ComponentDebug(r.name, "START")
+	r.wg.Add(1)
+	go func() {
+		err := r.server.ListenAndServe()
+		if err != nil && err.Error() != "http: Server closed" {
+			cclog.ComponentError(r.name, err.Error())
+		}
+		r.wg.Done()
+	}()
+}
+
+func (r *HttpReceiver) ServerHttp(w http.ResponseWriter, req *http.Request) {
+
+	// Check request method, only post method is handled
+	if req.Method != http.MethodPost {
+		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	// Check basic authentication
+	if r.config.useBasicAuth {
+		username, password, ok := req.BasicAuth()
+		if !ok || username != r.config.Username || password != r.config.Password {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+	}
+	if r.sink != nil {
+		buf := make([]byte, 0, req.ContentLength)
+		len, err := req.Body.Read(buf)
+		if err == nil && len > 0 {
+			messages, err := lp.FromBytes(buf)
+			if err != nil {
+				msg := "ServerHttp: Failed to decode messages: " + err.Error()
+				cclog.ComponentError(r.name, msg)
+				http.Error(w, msg, http.StatusInternalServerError)
+				return
+			}
+			for _, y := range messages {
+				m, err := r.mp.ProcessMessage(y)
+				if err == nil && m != nil {
+					r.sink <- m
+				}
+			}
+		}
+	}
+
+	w.WriteHeader(http.StatusOK)
+}
+
+func (r *HttpReceiver) Close() {
+	r.server.Shutdown(context.Background())
+}
+
+func NewHttpReceiver(name string, config json.RawMessage) (Receiver, error) {
+	r := new(HttpReceiver)
+	err := r.Init(name, config)
+	return r, err
+}

receivers/httpReceiver.md

@@ -0,0 +1,43 @@
+## `http` receiver
+
+The `http` receiver can be used receive metrics through HTTP POST requests.
+
+### Configuration structure
+
+```json
+{
+  "<name>": {
+    "type": "http",
+    "address" : "",
+    "port" : "8080",
+    "path" : "/write",
+    "idle_timeout": "120s",
+    "username": "myUser",
+    "password": "myPW"
+  }
+}
+```
+
+- `type`: makes the receiver a `http` receiver
+- `address`: Listen address
+- `port`: Listen port
+- `path`: URL path for the write endpoint
+- `idle_timeout`: Maximum amount of time to wait for the next request when keep-alives are enabled should be larger than the measurement interval to keep the connection open
+- `keep_alives_enabled`: Controls whether HTTP keep-alives are enabled. By default, keep-alives are enabled.
+- `username`: username for basic authentication
+- `password`: password for basic authentication
+
+The HTTP endpoint listens to `http://<address>:<port>/<path>`
+
+### Debugging
+
+- Install [curl](https://curl.se/)
+- Use curl to send message to `http` receiver
+
+  ```bash
+  curl http://localhost:8080/write \
+  --user "myUser:myPW" \
+  --data \
+  "myMetric,hostname=myHost,type=hwthread,type-id=0,unit=Hz value=400000i 1694777161164284635
+  myMetric,hostname=myHost,type=hwthread,type-id=1,unit=Hz value=400001i 1694777161164284635"
+  ```

receivers/ipmiReceiver.go

@@ -0,0 +1,571 @@
+package receivers
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	"github.com/ClusterCockpit/cc-metric-collector/pkg/hostlist"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
+)
+
+type IPMIReceiverClientConfig struct {
+
+	// Hostname the IPMI service belongs to
+	Protocol         string            // Protocol / tool to use for IPMI sensor reading
+	DriverType       string            // Out of band IPMI driver
+	Fanout           int               // Maximum number of simultaneous IPMI connections
+	NumHosts         int               // Number of remote IPMI devices with the same configuration
+	IPMIHosts        string            // List of remote IPMI devices to communicate with
+	IPMI2HostMapping map[string]string // Mapping between IPMI device name and host name
+	Username         string            // User name to authenticate with
+	Password         string            // Password to use for authentication
+	CLIOptions       []string          // Additional command line options for ipmi-sensors
+	isExcluded       map[string]bool   // is metric excluded
+	mp               mp.MessageProcessor
+}
+
+type IPMIReceiver struct {
+	receiver
+	config struct {
+		defaultReceiverConfig
+		Interval time.Duration
+
+		// Client config for each IPMI hosts
+		ClientConfigs []IPMIReceiverClientConfig
+	}
+
+	// Storage for static information
+	//meta map[string]string
+
+	done chan bool      // channel to finish / stop IPMI receiver
+	wg   sync.WaitGroup // wait group for IPMI receiver
+	mp   mp.MessageProcessor
+}
+
+// doReadMetrics reads metrics from all configure IPMI hosts.
+func (r *IPMIReceiver) doReadMetric() {
+	for i := range r.config.ClientConfigs {
+		clientConfig := &r.config.ClientConfigs[i]
+		var cmd_options []string
+		if clientConfig.Protocol == "ipmi-sensors" {
+			cmd_options = append(cmd_options,
+				"--always-prefix",
+				"--sdr-cache-recreate",
+				// Attempt to interpret OEM data, such as event data, sensor readings, or general extra info
+				"--interpret-oem-data",
+				// Ignore not-available (i.e. N/A) sensors in output
+				"--ignore-not-available-sensors",
+				// Ignore unrecognized sensor events
+				"--ignore-unrecognized-events",
+				// Output fields in comma separated format
+				"--comma-separated-output",
+				// Do not output column headers
+				"--no-header-output",
+				// Output non-abbreviated units (e.g. 'Amps' instead of 'A').
+				// May aid in disambiguation of units (e.g. 'C' for Celsius or Coulombs).
+				"--non-abbreviated-units",
+				"--fanout", fmt.Sprint(clientConfig.Fanout),
+				"--driver-type", clientConfig.DriverType,
+				"--hostname", clientConfig.IPMIHosts,
+				"--username", clientConfig.Username,
+				"--password", clientConfig.Password,
+			)
+			cmd_options := append(cmd_options, clientConfig.CLIOptions...)
+
+			command := exec.Command("ipmi-sensors", cmd_options...)
+			stdout, _ := command.StdoutPipe()
+			errBuf := new(bytes.Buffer)
+			command.Stderr = errBuf
+
+			// start command
+			if err := command.Start(); err != nil {
+				cclog.ComponentError(
+					r.name,
+					fmt.Sprintf("doReadMetric(): Failed to start command \"%s\": %v", command.String(), err),
+				)
+				continue
+			}
+
+			// Read command output
+			const (
+				idxID = iota
+				idxName
+				idxType
+				idxReading
+				idxUnits
+				idxEvent
+			)
+			numPrefixRegex := regexp.MustCompile("^[[:digit:]][[:digit:]]-(.*)$")
+			scanner := bufio.NewScanner(stdout)
+			for scanner.Scan() {
+				// Read host
+				v1 := strings.Split(scanner.Text(), ": ")
+				if len(v1) != 2 {
+					continue
+				}
+				host, ok := clientConfig.IPMI2HostMapping[v1[0]]
+				if !ok {
+					continue
+				}
+
+				// Read sensors
+				v2 := strings.Split(v1[1], ",")
+				if len(v2) != 6 {
+					continue
+				}
+				// Skip sensors with non available sensor readings
+				if v2[idxReading] == "N/A" {
+					continue
+				}
+
+				metric := strings.ToLower(v2[idxType])
+				name := strings.ToLower(
+					strings.Replace(
+						strings.TrimSpace(
+							v2[idxName]), " ", "_", -1))
+				// remove prefix enumeration like 01-...
+				if v := numPrefixRegex.FindStringSubmatch(name); v != nil {
+					name = v[1]
+				}
+				unit := v2[idxUnits]
+				if unit == "Watts" {
+
+					// Power
+					metric = "power"
+					name = strings.TrimSuffix(name, "_power")
+					name = strings.TrimSuffix(name, "_pwr")
+					name = strings.TrimPrefix(name, "pwr_")
+				} else if metric == "voltage" &&
+					unit == "Volts" {
+
+					// Voltage
+					name = strings.TrimPrefix(name, "volt_")
+				} else if metric == "current" &&
+					unit == "Amps" {
+
+					// Current
+					unit = "Ampere"
+				} else if metric == "temperature" &&
+					unit == "degrees C" {
+
+					// Temperature
+					name = strings.TrimSuffix(name, "_temp")
+					unit = "degC"
+				} else if metric == "temperature" &&
+					unit == "degrees F" {
+
+					// Temperature
+					name = strings.TrimSuffix(name, "_temp")
+					unit = "degF"
+				} else if metric == "fan" && unit == "RPM" {
+
+					// Fan speed
+					metric = "fan_speed"
+					name = strings.TrimSuffix(name, "_tach")
+					name = strings.TrimPrefix(name, "spd_")
+				} else if (metric == "cooling device" ||
+					metric == "other units based sensor") &&
+					name == "system_air_flow" &&
+					unit == "CFM" {
+
+					// Air flow
+					metric = "air_flow"
+					name = strings.TrimSuffix(name, "_air_flow")
+					unit = "CubicFeetPerMinute"
+				} else if (metric == "processor" ||
+					metric == "other units based sensor") &&
+					(name == "cpu_utilization" ||
+						name == "io_utilization" ||
+						name == "mem_utilization" ||
+						name == "sys_utilization") &&
+					(unit == "unspecified" ||
+						unit == "%") {
+
+					// Utilization
+					metric = "utilization"
+					name = strings.TrimSuffix(name, "_utilization")
+					unit = "percent"
+				} else {
+					if false {
+						// Debug output for unprocessed metrics
+						fmt.Printf(
+							"host: '%s', metric: '%s', name: '%s', unit: '%s'\n",
+							host, metric, name, unit)
+					}
+					continue
+				}
+
+				// Skip excluded metrics
+				if clientConfig.isExcluded[metric] {
+					continue
+				}
+
+				// Parse sensor value
+				value, err := strconv.ParseFloat(v2[idxReading], 64)
+				if err != nil {
+					continue
+				}
+
+				y, err := lp.NewMessage(
+					metric,
+					map[string]string{
+						"hostname": host,
+						"type":     "node",
+						"name":     name,
+					},
+					map[string]string{
+						"source": r.name,
+						"group":  "IPMI",
+						"unit":   unit,
+					},
+					map[string]interface{}{
+						"value": value,
+					},
+					time.Now())
+				if err == nil {
+					mc, err := clientConfig.mp.ProcessMessage(y)
+					if err == nil && mc != nil {
+						m, err := r.mp.ProcessMessage(mc)
+						if err == nil && m != nil {
+							r.sink <- m
+						}
+					}
+
+				}
+			}
+
+			// Wait for command end
+			if err := command.Wait(); err != nil {
+				errMsg, _ := io.ReadAll(errBuf)
+				cclog.ComponentError(
+					r.name,
+					fmt.Sprintf("doReadMetric(): Failed to wait for the end of command \"%s\": %v\n",
+						strings.Replace(command.String(), clientConfig.Password, "<PW>", -1), err),
+					fmt.Sprintf("doReadMetric(): command stderr: \"%s\"\n", string(errMsg)),
+				)
+			}
+		}
+	}
+}
+
+func (r *IPMIReceiver) Start() {
+	cclog.ComponentDebug(r.name, "START")
+
+	// Start IPMI receiver
+	r.wg.Add(1)
+	go func() {
+		defer r.wg.Done()
+
+		// Create ticker
+		ticker := time.NewTicker(r.config.Interval)
+		defer ticker.Stop()
+
+		for {
+			r.doReadMetric()
+
+			select {
+			case tickerTime := <-ticker.C:
+				// Check if we missed the ticker event
+				if since := time.Since(tickerTime); since > 5*time.Second {
+					cclog.ComponentInfo(r.name, "Missed ticker event for more then", since)
+				}
+
+				// process ticker event -> continue
+				continue
+			case <-r.done:
+				// process done event
+				return
+			}
+		}
+	}()
+
+	cclog.ComponentDebug(r.name, "STARTED")
+}
+
+// Close receiver: close network connection, close files, close libraries, ...
+func (r *IPMIReceiver) Close() {
+	cclog.ComponentDebug(r.name, "CLOSE")
+
+	// Send the signal and wait
+	close(r.done)
+	r.wg.Wait()
+
+	cclog.ComponentDebug(r.name, "DONE")
+}
+
+// NewIPMIReceiver creates a new instance of the redfish receiver
+// Initialize the receiver by giving it a name and reading in the config JSON
+func NewIPMIReceiver(name string, config json.RawMessage) (Receiver, error) {
+	var err error
+	r := new(IPMIReceiver)
+
+	// Config options from config file
+	configJSON := struct {
+		defaultReceiverConfig
+
+		// How often the IPMI sensor metrics should be read and send to the sink (default: 30 s)
+		IntervalString string `json:"interval,omitempty"`
+
+		// Maximum number of simultaneous IPMI connections (default: 64)
+		Fanout int `json:"fanout,omitempty"`
+
+		// Out of band IPMI driver (default: LAN_2_0)
+		DriverType string `json:"driver_type,omitempty"`
+
+		// Default client username, password and endpoint
+		Username *string `json:"username"` // User name to authenticate with
+		Password *string `json:"password"` // Password to use for authentication
+		Endpoint *string `json:"endpoint"` // URL of the IPMI device
+
+		// Globally excluded metrics
+		ExcludeMetrics []string `json:"exclude_metrics,omitempty"`
+
+		ClientConfigs []struct {
+			Fanout     int     `json:"fanout,omitempty"`      // Maximum number of simultaneous IPMI connections (default: 64)
+			DriverType string  `json:"driver_type,omitempty"` // Out of band IPMI driver (default: LAN_2_0)
+			HostList   string  `json:"host_list"`             // List of hosts with the same client configuration
+			Username   *string `json:"username"`              // User name to authenticate with
+			Password   *string `json:"password"`              // Password to use for authentication
+			Endpoint   *string `json:"endpoint"`              // URL of the IPMI service
+
+			// Per client excluded metrics
+			ExcludeMetrics []string `json:"exclude_metrics,omitempty"`
+
+			// Additional command line options for ipmi-sensors
+			CLIOptions       []string        `json:"cli_options,omitempty"`
+			MessageProcessor json.RawMessage `json:"process_messages,omitempty"`
+		} `json:"client_config"`
+	}{
+		// Set defaults values
+		// Allow overwriting these defaults by reading config JSON
+		Fanout:         64,
+		DriverType:     "LAN_2_0",
+		IntervalString: "30s",
+	}
+
+	// Set name of IPMIReceiver
+	r.name = fmt.Sprintf("IPMIReceiver(%s)", name)
+
+	// Create done channel
+	r.done = make(chan bool)
+
+	p, err := mp.NewMessageProcessor()
+	if err != nil {
+		return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
+	}
+	r.mp = p
+
+	// Set static information
+	//r.meta = map[string]string{"source": r.name}
+	r.mp.AddAddMetaByCondition("true", "source", r.name)
+
+	// Read the IPMI receiver specific JSON config
+	if len(config) > 0 {
+		d := json.NewDecoder(bytes.NewReader(config))
+		d.DisallowUnknownFields()
+		if err := d.Decode(&configJSON); err != nil {
+			cclog.ComponentError(r.name, "Error reading config:", err.Error())
+			return nil, err
+		}
+	}
+
+	if len(r.config.MessageProcessor) > 0 {
+		err = r.mp.FromConfigJSON(r.config.MessageProcessor)
+		if err != nil {
+			return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+		}
+	}
+	// Convert interval string representation to duration
+
+	r.config.Interval, err = time.ParseDuration(configJSON.IntervalString)
+	if err != nil {
+		err := fmt.Errorf(
+			"failed to parse duration string interval='%s': %w",
+			configJSON.IntervalString,
+			err,
+		)
+		cclog.Error(r.name, err)
+		return nil, err
+	}
+
+	// Create client config from JSON config
+	totalNumHosts := 0
+	for i := range configJSON.ClientConfigs {
+		clientConfigJSON := &configJSON.ClientConfigs[i]
+
+		var endpoint string
+		if clientConfigJSON.Endpoint != nil {
+			endpoint = *clientConfigJSON.Endpoint
+		} else if configJSON.Endpoint != nil {
+			endpoint = *configJSON.Endpoint
+		} else {
+			err := fmt.Errorf("client config number %v requires endpoint", i)
+			cclog.ComponentError(r.name, err)
+			return nil, err
+		}
+
+		fanout := configJSON.Fanout
+		if clientConfigJSON.Fanout != 0 {
+			fanout = clientConfigJSON.Fanout
+		}
+
+		driverType := configJSON.DriverType
+		if clientConfigJSON.DriverType != "" {
+			driverType = clientConfigJSON.DriverType
+		}
+		if driverType != "LAN" && driverType != "LAN_2_0" {
+			err := fmt.Errorf("client config number %v has invalid driver type %s", i, driverType)
+			cclog.ComponentError(r.name, err)
+			return nil, err
+		}
+
+		var protocol string
+		var host_pattern string
+		if e := strings.Split(endpoint, "://"); len(e) == 2 {
+			protocol = e[0]
+			host_pattern = e[1]
+		} else {
+			err := fmt.Errorf("client config number %v has invalid endpoint %s", i, endpoint)
+			cclog.ComponentError(r.name, err)
+			return nil, err
+		}
+
+		var username string
+		if clientConfigJSON.Username != nil {
+			username = *clientConfigJSON.Username
+		} else if configJSON.Username != nil {
+			username = *configJSON.Username
+		} else {
+			err := fmt.Errorf("client config number %v requires username", i)
+			cclog.ComponentError(r.name, err)
+			return nil, err
+		}
+
+		var password string
+		if clientConfigJSON.Password != nil {
+			password = *clientConfigJSON.Password
+		} else if configJSON.Password != nil {
+			password = *configJSON.Password
+		} else {
+			err := fmt.Errorf("client config number %v requires password", i)
+			cclog.ComponentError(r.name, err)
+			return nil, err
+		}
+
+		// Create mapping between IPMI host name and node host name
+		// This also guaranties that all IPMI host names are unique
+		ipmi2HostMapping := make(map[string]string)
+		hostList, err := hostlist.Expand(clientConfigJSON.HostList)
+		if err != nil {
+			err := fmt.Errorf("client config number %d failed to parse host list %s: %v",
+				i, clientConfigJSON.HostList, err)
+			cclog.ComponentError(r.name, err)
+			return nil, err
+		}
+		for _, host := range hostList {
+			ipmiHost := strings.Replace(host_pattern, "%h", host, -1)
+			ipmi2HostMapping[ipmiHost] = host
+		}
+
+		numHosts := len(ipmi2HostMapping)
+		totalNumHosts += numHosts
+		ipmiHostList := make([]string, 0, numHosts)
+		for ipmiHost := range ipmi2HostMapping {
+			ipmiHostList = append(ipmiHostList, ipmiHost)
+		}
+
+		// Additional command line options
+		for _, v := range clientConfigJSON.CLIOptions {
+			switch {
+			case v == "-u" || strings.HasPrefix(v, "--username"):
+				err := fmt.Errorf("client config number %v: do not set username in cli_options. Use json config username instead", i)
+				cclog.ComponentError(r.name, err)
+				return nil, err
+			case v == "-p" || strings.HasPrefix(v, "--password"):
+				err := fmt.Errorf("client config number %v: do not set password in cli_options. Use json config password instead", i)
+				cclog.ComponentError(r.name, err)
+				return nil, err
+			case v == "-h" || strings.HasPrefix(v, "--hostname"):
+				err := fmt.Errorf("client config number %v: do not set hostname in cli_options. Use json config host_list instead", i)
+				cclog.ComponentError(r.name, err)
+				return nil, err
+			case v == "-D" || strings.HasPrefix(v, "--driver-type"):
+				err := fmt.Errorf("client config number %v: do not set driver type in cli_options. Use json config driver_type instead", i)
+				cclog.ComponentError(r.name, err)
+				return nil, err
+			case v == "-F" || strings.HasPrefix(v, " --fanout"):
+				err := fmt.Errorf("client config number %v: do not set fanout in cli_options. Use json config fanout instead", i)
+				cclog.ComponentError(r.name, err)
+				return nil, err
+			case v == "--always-prefix" ||
+				v == "--sdr-cache-recreate" ||
+				v == "--interpret-oem-data" ||
+				v == "--ignore-not-available-sensors" ||
+				v == "--ignore-unrecognized-events" ||
+				v == "--comma-separated-output" ||
+				v == "--no-header-output" ||
+				v == "--non-abbreviated-units":
+				err := fmt.Errorf("client config number %v: Do not use option %s in cli_options, it is used internally", i, v)
+				cclog.ComponentError(r.name, err)
+				return nil, err
+			}
+		}
+		cliOptions := make([]string, 0)
+		cliOptions = append(cliOptions, clientConfigJSON.CLIOptions...)
+
+		// Is metrics excluded globally or per client
+		isExcluded := make(map[string]bool)
+		for _, key := range clientConfigJSON.ExcludeMetrics {
+			isExcluded[key] = true
+		}
+		for _, key := range configJSON.ExcludeMetrics {
+			isExcluded[key] = true
+		}
+		p, err := mp.NewMessageProcessor()
+		if err != nil {
+			return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
+		}
+		if len(clientConfigJSON.MessageProcessor) > 0 {
+			err = p.FromConfigJSON(clientConfigJSON.MessageProcessor)
+			if err != nil {
+				return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+			}
+		}
+
+		r.config.ClientConfigs = append(
+			r.config.ClientConfigs,
+			IPMIReceiverClientConfig{
+				Protocol:         protocol,
+				Fanout:           fanout,
+				DriverType:       driverType,
+				NumHosts:         numHosts,
+				IPMIHosts:        strings.Join(ipmiHostList, ","),
+				IPMI2HostMapping: ipmi2HostMapping,
+				Username:         username,
+				Password:         password,
+				CLIOptions:       cliOptions,
+				isExcluded:       isExcluded,
+				mp:               p,
+			})
+	}
+
+	if totalNumHosts == 0 {
+		err := fmt.Errorf("at least one IPMI host config is required")
+		cclog.ComponentError(r.name, err)
+		return nil, err
+	}
+
+	cclog.ComponentInfo(r.name, "monitoring", totalNumHosts, "IPMI hosts")
+	return r, nil
+}

receivers/ipmiReceiver.md

@@ -0,0 +1,48 @@
+## IPMI Receiver
+
+The IPMI Receiver uses `ipmi-sensors` from the [FreeIPMI](https://www.gnu.org/software/freeipmi/) project to read IPMI sensor readings and sensor data repository (SDR) information. The available metrics depend on the sensors provided by the hardware vendor but typically contain temperature, fan speed, voltage and power metrics.
+
+### Configuration structure
+
+```json
+{
+    "<IPMI receiver name>": {
+        "type": "ipmi",
+        "interval": "30s",
+        "fanout": 256,
+        "username": "<Username>",
+        "password": "<Password>",
+        "endpoint": "ipmi-sensors://%h-bmc",
+        "exclude_metrics": [ "fan_speed", "voltage" ],
+        "client_config": [
+            {
+                "host_list": "n[1,2-4]"
+            },
+            {
+                "host_list": "n[5-6]",
+                "driver_type": "LAN",
+                "cli_options": [ "--workaround-flags=..." ],
+                "password": "<Password 2>"
+            }
+        ]
+    }
+}
+```
+
+Global settings:
+
+- `interval`: How often the IPMI sensor metrics should be read and send to the sink (default: 30 s)
+
+Global and per IPMI device settings (per IPMI device settings overwrite the global settings):
+
+- `exclude_metrics`: list of excluded metrics e.g. fan_speed, power, temperature, utilization, voltage
+- `fanout`: Maximum number of simultaneous IPMI connections (default: 64)
+- `driver_type`: Out of band IPMI driver (default: LAN_2_0)
+- `username`: User name to authenticate with
+- `password`: Password to use for authentication
+- `endpoint`: URL of the IPMI device (placeholder `%h` gets replaced by the hostname)
+
+Per IPMI device settings:
+
+- `host_list`: List of hosts with the same client configuration
+- `cli_options`: Additional command line options for ipmi-sensors

receivers/metricReceiver.go

@@ -0,0 +1,45 @@
+package receivers
+
+import (
+	"encoding/json"
+
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
+)
+
+type defaultReceiverConfig struct {
+	Type             string          `json:"type"`
+	MessageProcessor json.RawMessage `json:"process_messages,omitempty"`
+}
+
+// Receiver configuration: Listen address, port
+type ReceiverConfig struct {
+	Addr         string `json:"address"`
+	Port         string `json:"port"`
+	Database     string `json:"database"`
+	Organization string `json:"organization,omitempty"`
+	Type         string `json:"type"`
+}
+
+type receiver struct {
+	name string
+	sink chan lp.CCMessage
+	mp   mp.MessageProcessor
+}
+
+type Receiver interface {
+	Start()
+	Close()                         // Close / finish metric receiver
+	Name() string                   // Name of the metric receiver
+	SetSink(sink chan lp.CCMessage) // Set sink channel
+}
+
+// Name returns the name of the metric receiver
+func (r *receiver) Name() string {
+	return r.name
+}
+
+// SetSink set the sink channel
+func (r *receiver) SetSink(sink chan lp.CCMessage) {
+	r.sink = sink
+}

receivers/natsReceiver.go

@@ -0,0 +1,127 @@
+package receivers
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
+	nats "github.com/nats-io/nats.go"
+)
+
+type NatsReceiverConfig struct {
+	defaultReceiverConfig
+	Addr     string `json:"address"`
+	Port     string `json:"port"`
+	Subject  string `json:"subject"`
+	User     string `json:"user,omitempty"`
+	Password string `json:"password,omitempty"`
+	NkeyFile string `json:"nkey_file,omitempty"`
+}
+
+type NatsReceiver struct {
+	receiver
+	nc *nats.Conn
+	//meta   map[string]string
+	config NatsReceiverConfig
+}
+
+// Start subscribes to the configured NATS subject
+// Messages wil be handled by r._NatsReceive
+func (r *NatsReceiver) Start() {
+	cclog.ComponentDebug(r.name, "START")
+	r.nc.Subscribe(r.config.Subject, r._NatsReceive)
+}
+
+// _NatsReceive receives subscribed messages from the NATS server
+func (r *NatsReceiver) _NatsReceive(m *nats.Msg) {
+
+	if r.sink != nil {
+		messages, err := lp.FromBytes(m.Data)
+		if err != nil {
+			msg := "_NatsReceive: Failed to decode messages: " + err.Error()
+			cclog.ComponentError(r.name, msg)
+		}
+		for _, y := range messages {
+			m, err := r.mp.ProcessMessage(y)
+			if err == nil && m != nil && r.sink != nil {
+				r.sink <- m
+			}
+		}
+	}
+}
+
+// Close closes the connection to the NATS server
+func (r *NatsReceiver) Close() {
+	if r.nc != nil {
+		cclog.ComponentDebug(r.name, "CLOSE")
+		r.nc.Close()
+	}
+}
+
+// NewNatsReceiver creates a new Receiver which subscribes to messages from a NATS server
+func NewNatsReceiver(name string, config json.RawMessage) (Receiver, error) {
+	var uinfo nats.Option = nil
+	r := new(NatsReceiver)
+	r.name = fmt.Sprintf("NatsReceiver(%s)", name)
+
+	// Read configuration file, allow overwriting default config
+	r.config.Addr = "localhost"
+	r.config.Port = "4222"
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &r.config)
+		if err != nil {
+			cclog.ComponentError(r.name, "Error reading config:", err.Error())
+			return nil, err
+		}
+	}
+	if len(r.config.Addr) == 0 ||
+		len(r.config.Port) == 0 ||
+		len(r.config.Subject) == 0 {
+		return nil, errors.New("not all configuration variables set required by NatsReceiver")
+	}
+	p, err := mp.NewMessageProcessor()
+	if err != nil {
+		return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
+	}
+	r.mp = p
+	if len(r.config.MessageProcessor) > 0 {
+		err = r.mp.FromConfigJSON(r.config.MessageProcessor)
+		if err != nil {
+			return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+		}
+	}
+
+	// Set metadata
+	// r.meta = map[string]string{
+	// 	"source": r.name,
+	// }
+	r.mp.AddAddMetaByCondition("true", "source", r.name)
+
+	if len(r.config.User) > 0 && len(r.config.Password) > 0 {
+		uinfo = nats.UserInfo(r.config.User, r.config.Password)
+	} else if len(r.config.NkeyFile) > 0 {
+		_, err := os.Stat(r.config.NkeyFile)
+		if err == nil {
+			uinfo = nats.UserCredentials(r.config.NkeyFile)
+		} else {
+			cclog.ComponentError(r.name, "NKEY file", r.config.NkeyFile, "does not exist: %v", err.Error())
+			return nil, err
+		}
+	}
+
+	// Connect to NATS server
+	url := fmt.Sprintf("nats://%s:%s", r.config.Addr, r.config.Port)
+	cclog.ComponentDebug(r.name, "NewNatsReceiver", url, "Subject", r.config.Subject)
+	if nc, err := nats.Connect(url, uinfo); err == nil {
+		r.nc = nc
+	} else {
+		r.nc = nil
+		return nil, err
+	}
+
+	return r, nil
+}

receivers/natsReceiver.md

@@ -0,0 +1,56 @@
+## `nats` receiver
+
+The `nats` receiver can be used receive metrics from the NATS network. The `nats` receiver subscribes to the topic `database` and listens on `address` and `port` for metrics in the InfluxDB line protocol.
+
+### Configuration structure
+
+```json
+{
+  "<name>": {
+    "type": "nats",
+    "address" : "nats-server.example.org",
+    "port" : "4222",
+    "subject" : "subject",
+    "user": "natsuser",
+    "password": "natssecret",
+    "nkey_file": "/path/to/nkey_file"
+  }
+}
+```
+
+- `type`: makes the receiver a `nats` receiver
+- `address`: Address of the NATS control server
+- `port`: Port of the NATS control server
+- `subject`: Subscribes to this subject and receive metrics
+- `user`: Connect to nats using this user
+- `password`: Connect to nats using this password
+- `nkey_file`: Path to credentials file with NKEY
+
+### Debugging
+
+- Install NATS server and command line client
+- Start NATS server
+
+  ```bash
+  nats-server --net nats-server.example.org --port 4222
+  ```
+
+- Check NATS server works as expected
+
+  ```bash
+  nats --server=nats-server-db.example.org:4222 server check
+  ```
+
+- Use NATS command line client to subscribe to all messages
+
+  ```bash
+  nats --server=nats-server-db.example.org:4222 sub ">"
+  ```
+
+- Use NATS command line client to send message to NATS receiver
+
+  ```bash
+  nats --server=nats-server-db.example.org:4222 pub subject \
+  "myMetric,hostname=myHost,type=hwthread,type-id=0,unit=Hz value=400000i 1694777161164284635
+  myMetric,hostname=myHost,type=hwthread,type-id=1,unit=Hz value=400001i 1694777161164284635"
+  ```

receivers/prometheusReceiver.go

@@ -0,0 +1,122 @@
+package receivers
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+)
+
+type PrometheusReceiverConfig struct {
+	defaultReceiverConfig
+	Addr     string `json:"address"`
+	Port     string `json:"port"`
+	Path     string `json:"path"`
+	Interval string `json:"interval"`
+	SSL      bool   `json:"ssl"`
+}
+
+type PrometheusReceiver struct {
+	receiver
+	meta     map[string]string
+	config   PrometheusReceiverConfig
+	interval time.Duration
+	done     chan bool
+	wg       sync.WaitGroup
+	ticker   *time.Ticker
+	uri      string
+}
+
+func (r *PrometheusReceiver) Start() {
+	cclog.ComponentDebug(r.name, "START", r.uri)
+	r.wg.Add(1)
+
+	r.ticker = time.NewTicker(r.interval)
+	go func() {
+		for {
+			select {
+			case <-r.done:
+				r.wg.Done()
+				return
+			case t := <-r.ticker.C:
+				resp, err := http.Get(r.uri)
+				if err != nil {
+					log.Fatal(err)
+				}
+				defer resp.Body.Close()
+
+				scanner := bufio.NewScanner(resp.Body)
+				for scanner.Scan() {
+					line := scanner.Text()
+					if strings.HasPrefix(line, "#") {
+						continue
+					}
+					lineSplit := strings.Fields(line)
+					// separate metric name from tags (labels in Prometheus)
+					tags := map[string]string{}
+					name := lineSplit[0]
+					if sindex := strings.Index(name, "{"); sindex >= 0 {
+						eindex := strings.Index(name, "}")
+						for _, kv := range strings.Split(name[sindex+1:eindex], ",") {
+							eq := strings.Index(kv, "=")
+							tags[kv[0:eq]] = strings.Trim(kv[eq+1:], "\"")
+						}
+						name = lineSplit[0][0:sindex]
+					}
+					value, err := strconv.ParseFloat(lineSplit[1], 64)
+					if err == nil {
+						y, err := lp.NewMessage(name, tags, r.meta, map[string]interface{}{"value": value}, t)
+						if err == nil {
+							r.sink <- y
+						}
+					}
+				}
+			}
+		}
+	}()
+}
+
+func (r *PrometheusReceiver) Close() {
+	cclog.ComponentDebug(r.name, "CLOSE")
+	r.done <- true
+	r.wg.Wait()
+}
+
+func NewPrometheusReceiver(name string, config json.RawMessage) (Receiver, error) {
+	r := new(PrometheusReceiver)
+	r.name = fmt.Sprintf("PrometheusReceiver(%s)", name)
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &r.config)
+		if err != nil {
+			cclog.ComponentError(r.name, "Error reading config:", err.Error())
+			return nil, err
+		}
+	}
+	if len(r.config.Addr) == 0 ||
+		len(r.config.Port) == 0 ||
+		len(r.config.Interval) == 0 {
+		return nil, errors.New("not all configuration variables set required by PrometheusReceiver (address and port)")
+	}
+	if len(r.config.Interval) > 0 {
+		t, err := time.ParseDuration(r.config.Interval)
+		if err == nil {
+			r.interval = t
+		}
+	}
+	r.meta = map[string]string{"source": r.name}
+	proto := "http"
+	if r.config.SSL {
+		proto = "https"
+	}
+	r.uri = fmt.Sprintf("%s://%s:%s/%s", proto, r.config.Addr, r.config.Port, r.config.Path)
+	return r, nil
+}

receivers/prometheusReceiver.md

@@ -0,0 +1,27 @@
+## `prometheus` receiver
+
+The `prometheus` receiver can be used to scrape the metrics of a single `prometheus` client. It does **not** use any official Golang library but making simple HTTP get requests and parse the response.
+
+### Configuration structure
+
+```json
+{
+  "<name>": {
+    "type": "prometheus",
+    "address" : "testpromhost",
+    "port" : "12345",
+    "path" : "/prometheus",
+    "interval": "5s",
+    "ssl" : true,
+  }
+}
+```
+
+- `type`: makes the receiver a `prometheus` receiver
+- `address`: Hostname or IP of the Prometheus agent
+- `port`: Port of Prometheus agent
+- `path`: Path to the Prometheus endpoint
+- `interval`: Scrape the Prometheus endpoint in this interval (default '5s')
+- `ssl`: Use SSL or not
+
+The receiver requests data from `http(s)://<address>:<port>/<path>`.

receivers/receiveManager.go

@@ -0,0 +1,122 @@
+package receivers
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+)
+
+var AvailableReceivers = map[string]func(name string, config json.RawMessage) (Receiver, error){
+	"http":    NewHttpReceiver,
+	"ipmi":    NewIPMIReceiver,
+	"nats":    NewNatsReceiver,
+	"redfish": NewRedfishReceiver,
+}
+
+type receiveManager struct {
+	inputs []Receiver
+	output chan lp.CCMessage
+	config []json.RawMessage
+}
+
+type ReceiveManager interface {
+	Init(wg *sync.WaitGroup, receiverConfigFile string) error
+	AddInput(name string, rawConfig json.RawMessage) error
+	AddOutput(output chan lp.CCMessage)
+	Start()
+	Close()
+}
+
+func (rm *receiveManager) Init(wg *sync.WaitGroup, receiverConfigFile string) error {
+	// Initialize struct fields
+	rm.inputs = make([]Receiver, 0)
+	rm.output = nil
+	rm.config = make([]json.RawMessage, 0)
+
+	configFile, err := os.Open(receiverConfigFile)
+	if err != nil {
+		cclog.ComponentError("ReceiveManager", err.Error())
+		return err
+	}
+	defer configFile.Close()
+	jsonParser := json.NewDecoder(configFile)
+	var rawConfigs map[string]json.RawMessage
+	err = jsonParser.Decode(&rawConfigs)
+	if err != nil {
+		cclog.ComponentError("ReceiveManager", err.Error())
+		return err
+	}
+	for name, raw := range rawConfigs {
+		rm.AddInput(name, raw)
+	}
+
+	return nil
+}
+
+func (rm *receiveManager) Start() {
+	cclog.ComponentDebug("ReceiveManager", "START")
+
+	for _, r := range rm.inputs {
+		cclog.ComponentDebug("ReceiveManager", "START", r.Name())
+		r.Start()
+	}
+	cclog.ComponentDebug("ReceiveManager", "STARTED")
+}
+
+func (rm *receiveManager) AddInput(name string, rawConfig json.RawMessage) error {
+	var config defaultReceiverConfig
+	err := json.Unmarshal(rawConfig, &config)
+	if err != nil {
+		cclog.ComponentError("ReceiveManager", "SKIP", config.Type, "JSON config error:", err.Error())
+		return err
+	}
+	if config.Type == "" {
+		cclog.ComponentError("ReceiveManager", "SKIP", "JSON config for receiver", name, "does not contain a receiver type")
+		return fmt.Errorf("JSON config for receiver %s does not contain a receiver type", name)
+	}
+	if _, found := AvailableReceivers[config.Type]; !found {
+		cclog.ComponentError("ReceiveManager", "SKIP", "unknown receiver type:", config.Type)
+		return fmt.Errorf("unknown receiver type: %s", config.Type)
+	}
+	r, err := AvailableReceivers[config.Type](name, rawConfig)
+	if err != nil {
+		cclog.ComponentError("ReceiveManager", "SKIP", name, "initialization failed:", err.Error())
+		return err
+	}
+	rm.inputs = append(rm.inputs, r)
+	rm.config = append(rm.config, rawConfig)
+	cclog.ComponentDebug("ReceiveManager", "ADD RECEIVER", r.Name())
+	return nil
+}
+
+func (rm *receiveManager) AddOutput(output chan lp.CCMessage) {
+	rm.output = output
+	for _, r := range rm.inputs {
+		r.SetSink(rm.output)
+	}
+}
+
+func (rm *receiveManager) Close() {
+	cclog.ComponentDebug("ReceiveManager", "CLOSE")
+
+	// Close all receivers
+	for _, r := range rm.inputs {
+		cclog.ComponentDebug("ReceiveManager", "CLOSE", r.Name())
+		r.Close()
+	}
+
+	cclog.ComponentDebug("ReceiveManager", "DONE")
+}
+
+func New(wg *sync.WaitGroup, receiverConfigFile string) (ReceiveManager, error) {
+	r := new(receiveManager)
+	err := r.Init(wg, receiverConfigFile)
+	if err != nil {
+		return nil, err
+	}
+	return r, err
+}

receivers/redfishReceiver.md

@@ -0,0 +1,65 @@
+## Redfish receiver
+
+The Redfish receiver uses the [Redfish (specification)](https://www.dmtf.org/standards/redfish) to query thermal and power metrics. Thermal metrics may include various fan speeds and temperatures. Power metrics may include the current power consumption of various hardware components. It may also include the minimum, maximum and average power consumption of these components in a given time interval. The receiver will poll each configured redfish device once in a given interval. Multiple devices can be accessed in parallel to increase throughput.
+
+### Configuration structure
+
+```json
+{
+    "<redfish receiver name>": {
+        "type": "redfish",
+        "username": "<Username>",
+        "password": "<Password>",
+        "endpoint": "https://%h-bmc",
+        "exclude_metrics": [ "min_consumed_watts" ],
+        "client_config": [
+            {
+                "host_list": "n[1,2-4]"
+            },
+            {
+                "host_list": "n5",
+                "disable_power_metrics": true,
+                "disable_processor_metrics": true,
+                "disable_thermal_metrics": true
+            },
+            {
+                "host_list": "n6" ],
+                "username": "<Username 2>",
+                "password": "<Password 2>",
+                "endpoint": "https://%h-BMC",
+                "disable_sensor_metrics": true
+            }
+        ]
+    }
+}
+```
+
+Global settings:
+
+- `fanout`: Maximum number of simultaneous redfish connections (default: 64)
+- `interval`: How often the redfish power metrics should be read and send to the sink (default: 30 s)
+- `http_insecure`: Control whether a client verifies the server's certificate (default: true == do not verify server's certificate)
+- `http_timeout`: Time limit for requests made by this HTTP client (default: 10 s)
+
+Global and per redfish device settings (per redfish device settings overwrite the global settings):
+
+- `disable_power_metrics`:
+  disable collection of power metrics
+  (`/redfish/v1/Chassis/{ChassisId}/Power`)
+- `disable_processor_metrics`:
+  disable collection of processor metrics
+  (`/redfish/v1/Systems/{ComputerSystemId}/Processors/{ProcessorId}/ProcessorMetrics`)
+- `disable_sensors`:
+  disable collection of fan, power and thermal sensor metrics
+  (`/redfish/v1/Chassis/{ChassisId}/Sensors/{SensorId}`)
+- `disable_thermal_metrics`:
+  disable collection of thermal metrics
+  (`/redfish/v1/Chassis/{ChassisId}/Thermal`)
+- `exclude_metrics`: list of excluded metrics
+- `username`: User name to authenticate with
+- `password`: Password to use for authentication
+- `endpoint`: URL of the redfish service (placeholder `%h` gets replaced by the hostname)
+
+Per redfish device settings:
+
+- `host_list`: List of hosts with the same client configuration

receivers/sampleReceiver.go

@@ -0,0 +1,122 @@
+package receivers
+
+import (
+	"encoding/json"
+	"fmt"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
+)
+
+// SampleReceiver configuration: receiver type, listen address, port
+// The defaultReceiverConfig contains the keys 'type' and 'process_messages'
+type SampleReceiverConfig struct {
+	defaultReceiverConfig
+	Addr string `json:"address"`
+	Port string `json:"port"`
+}
+
+type SampleReceiver struct {
+	receiver
+	config SampleReceiverConfig
+
+	// Storage for static information
+	// Use in case of own go routine
+	// done chan bool
+	// wg   sync.WaitGroup
+}
+
+// Implement functions required for Receiver interface
+// Start(), Close()
+// See: metricReceiver.go
+
+func (r *SampleReceiver) Start() {
+	cclog.ComponentDebug(r.name, "START")
+
+	// Start server process like http.ListenAndServe()
+
+	// or use own go routine but always make sure it exits
+	// as soon as it gets the signal of the r.done channel
+	//
+	// r.done = make(chan bool)
+	// r.wg.Add(1)
+	// go func() {
+	//      defer r.wg.Done()
+	//
+	//      // Create ticker
+	//      ticker := time.NewTicker(30 * time.Second)
+	//      defer ticker.Stop()
+	//
+	//      for {
+	//          readMetric()
+	//          select {
+	//          case <-ticker.C:
+	//              // process ticker event -> continue
+	//              continue
+	//          case <-r.done:
+	//              return
+	//          }
+	//      }
+	// }()
+}
+
+// Close receiver: close network connection, close files, close libraries, ...
+func (r *SampleReceiver) Close() {
+	cclog.ComponentDebug(r.name, "CLOSE")
+
+	// Close server like http.Shutdown()
+
+	// in case of own go routine, send the signal and wait
+	// r.done <- true
+	// r.wg.Wait()
+}
+
+// New function to create a new instance of the receiver
+// Initialize the receiver by giving it a name and reading in the config JSON
+func NewSampleReceiver(name string, config json.RawMessage) (Receiver, error) {
+	r := new(SampleReceiver)
+
+	// Set name of SampleReceiver
+	// The name should be chosen in such a way that different instances of SampleReceiver can be distinguished
+	r.name = fmt.Sprintf("SampleReceiver(%s)", name)
+
+	// create new message processor
+	p, err := mp.NewMessageProcessor()
+	if err != nil {
+		cclog.ComponentError(r.name, "Initialization of message processor failed:", err.Error())
+		return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
+	}
+	r.mp = p
+	// Set static information
+	err = r.mp.AddAddMetaByCondition("true", "source", r.name)
+	if err != nil {
+		cclog.ComponentError(r.name, fmt.Sprintf("Failed to add static information source=%s:", r.name), err.Error())
+		return nil, fmt.Errorf("failed to add static information source=%s: %v", r.name, err.Error())
+	}
+
+	// Set defaults in r.config
+	// Allow overwriting these defaults by reading config JSON
+
+	// Read the sample receiver specific JSON config
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &r.config)
+		if err != nil {
+			cclog.ComponentError(r.name, "Error reading config:", err.Error())
+			return nil, err
+		}
+	}
+
+	// Add message processor config
+	if len(r.config.MessageProcessor) > 0 {
+		err = r.mp.FromConfigJSON(r.config.MessageProcessor)
+		if err != nil {
+			cclog.ComponentError(r.name, "Failed parsing JSON for message processor:", err.Error())
+			return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+		}
+	}
+
+	// Check that all required fields in the configuration are set
+	// Use 'if len(r.config.Option) > 0' for strings
+
+	return r, nil
+}

sinks/README.md

@@ -0,0 +1,108 @@
+# CCMetric sinks
+
+This folder contains the SinkManager and sink implementations for the cc-metric-collector.
+
+# Available sinks:
+- [`stdout`](./stdoutSink.md): Print all metrics to `stdout`, `stderr` or a file
+- [`http`](./httpSink.md): Send metrics to an HTTP server as POST requests
+- [`influxdb`](./influxSink.md): Send metrics to an [InfluxDB](https://www.influxdata.com/products/influxdb/) database
+- [`influxasync`](./influxAsyncSink.md): Send metrics to an [InfluxDB](https://www.influxdata.com/products/influxdb/) database with non-blocking write API
+- [`nats`](./natsSink.md): Publish metrics to the [NATS](https://nats.io/) network overlay system
+- [`ganglia`](./gangliaSink.md): Publish metrics in the [Ganglia Monitoring System](http://ganglia.info/) using the `gmetric` CLI tool
+- [`libganglia`](./libgangliaSink.md): Publish metrics in the [Ganglia Monitoring System](http://ganglia.info/) directly using `libganglia.so`
+- [`prometeus`](./prometheusSink.md): Publish metrics for the [Prometheus Monitoring System](https://prometheus.io/)
+
+# Configuration
+
+The configuration file for the sinks is a list of configurations. The `type` field in each specifies which sink to initialize.
+
+```json
+{
+  "mystdout" : {
+    "type" : "stdout",
+    "meta_as_tags" : [
+    	"unit"
+    ]
+  },
+  "metricstore" : {
+    "type" : "http",
+    "host" : "localhost",
+    "port" : "4123",
+    "database" : "ccmetric",
+    "password" : "<jwt token>"
+  }
+}
+```
+
+
+
+
+# Contributing own sinks
+A sink contains five functions and is derived from the type `sink`:
+* `Init(name string, config json.RawMessage) error`
+* `Write(point CCMetric) error`
+* `Flush() error`
+* `Close()`
+* `New<Typename>(name string, config json.RawMessage) (Sink, error)` (calls the `Init()` function)
+
+The data structures should be set up in `Init()` like opening a file or server connection. The `Write()` function writes/sends the data. For non-blocking sinks, the `Flush()` method tells the sink to drain its internal buffers. The `Close()` function should tear down anything created in `Init()`.
+
+Finally, the sink needs to be registered in the `sinkManager.go`. There is a list of sinks called `AvailableSinks` which is a map (`sink_type_string` -> `pointer to sink interface`). Add a new entry with a descriptive name and the new sink.
+
+## Sample sink
+
+```go
+package sinks
+
+import (
+	"encoding/json"
+	"log"
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+)
+
+type SampleSinkConfig struct {
+	defaultSinkConfig  // defines JSON tags for 'name' and 'meta_as_tags'
+}
+
+type SampleSink struct {
+	sink              // declarate 'name' and 'meta_as_tags'
+	config StdoutSinkConfig // entry point to the SampleSinkConfig
+}
+
+// Initialize the sink by giving it a name and reading in the config JSON
+func (s *SampleSink) Init(name string, config json.RawMessage) error {
+	s.name = fmt.Sprintf("SampleSink(%s)", name)   // Always specify a name here
+  // Read in the config JSON
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &s.config)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Code to submit a single CCMetric to the sink
+func (s *SampleSink) Write(point lp.CCMetric) error {
+	log.Print(point)
+	return nil
+}
+
+// If the sink uses batched sends internally, you can tell to flush its buffers
+func (s *SampleSink) Flush() error {
+	return nil
+}
+
+
+// Close sink: close network connection, close files, close libraries, ...
+func (s *SampleSink) Close() {}
+
+
+// New function to create a new instance of the sink
+func NewSampleSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(SampleSink)
+	err := s.Init(name, config)
+	return s, err
+}
+
+```

sinks/gangliaCommon.go

@@ -0,0 +1,269 @@
+package sinks
+
+import (
+	"fmt"
+	"strings"
+
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+)
+
+func GangliaMetricName(point lp.CCMessage) string {
+	name := point.Name()
+	metricType, typeOK := point.GetTag("type")
+	metricTid, tidOk := point.GetTag("type-id")
+	gangliaType := metricType + metricTid
+	if strings.Contains(name, metricType) && tidOk {
+		name = strings.Replace(name, metricType, gangliaType, -1)
+	} else if typeOK && tidOk {
+		name = metricType + metricTid + "_" + name
+	} else if point.HasTag("device") {
+		device, _ := point.GetTag("device")
+		name = name + "_" + device
+	}
+
+	return name
+}
+
+func GangliaMetricRename(name string) string {
+	if name == "net_bytes_in" {
+		return "bytes_in"
+	} else if name == "net_bytes_out" {
+		return "bytes_out"
+	} else if name == "net_pkts_in" {
+		return "pkts_in"
+	} else if name == "net_pkts_out" {
+		return "pkts_out"
+	} else if name == "cpu_iowait" {
+		return "cpu_wio"
+	}
+	return name
+}
+
+func GangliaSlopeType(point lp.CCMessage) uint {
+	name := point.Name()
+	if name == "mem_total" || name == "swap_total" {
+		return 0
+	}
+	return 3
+}
+
+const DEFAULT_GANGLIA_METRIC_TMAX = 300
+const DEFAULT_GANGLIA_METRIC_SLOPE = "both"
+
+type GangliaMetric struct {
+	Name  string
+	Type  string
+	Slope string
+	Tmax  int
+	Unit  string
+}
+
+type GangliaMetricGroup struct {
+	Name    string
+	Metrics []GangliaMetric
+}
+
+var CommonGangliaMetrics = []GangliaMetricGroup{
+	{
+		Name: "memory",
+		Metrics: []GangliaMetric{
+			{"mem_total", "float", "zero", 1200, "KB"},
+			{"swap_total", "float", "zero", 1200, "KB"},
+			{"mem_free", "float", "both", 180, "KB"},
+			{"mem_shared", "float", "both", 180, "KB"},
+			{"mem_buffers", "float", "both", 180, "KB"},
+			{"mem_cached", "float", "both", 180, "KB"},
+			{"swap_free", "float", "both", 180, "KB"},
+			{"mem_sreclaimable", "float", "both", 180, "KB"},
+			{"mem_slab", "float", "both", 180, "KB"},
+		},
+	},
+	{
+		Name: "cpu",
+		Metrics: []GangliaMetric{
+			{"cpu_num", "uint32", "zero", 1200, "CPUs"},
+			{"cpu_speed", "uint32", "zero", 1200, "MHz"},
+			{"cpu_user", "float", "both", 90, "%"},
+			{"cpu_nice", "float", "both", 90, "%"},
+			{"cpu_system", "float", "both", 90, "%"},
+			{"cpu_idle", "float", "both", 3800, "%"},
+			{"cpu_aidle", "float", "both", 90, "%"},
+			{"cpu_wio", "float", "both", 90, "%"},
+			{"cpu_intr", "float", "both", 90, "%"},
+			{"cpu_sintr", "float", "both", 90, "%"},
+			{"cpu_steal", "float", "both", 90, "%"},
+			{"cpu_guest", "float", "both", 90, "%"},
+			{"cpu_gnice", "float", "both", 90, "%"},
+		},
+	},
+	{
+		Name: "load",
+		Metrics: []GangliaMetric{
+			{"load_one", "float", "both", 70, ""},
+			{"load_five", "float", "both", 325, ""},
+			{"load_fifteen", "float", "both", 950, ""},
+		},
+	},
+	{
+		Name: "disk",
+		Metrics: []GangliaMetric{
+			{"disk_total", "double", "both", 1200, "GB"},
+			{"disk_free", "double", "both", 180, "GB"},
+			{"part_max_used", "float", "both", 180, "%"},
+		},
+	},
+	{
+		Name: "network",
+		Metrics: []GangliaMetric{
+			{"bytes_out", "float", "both", 300, "bytes/sec"},
+			{"bytes_in", "float", "both", 300, "bytes/sec"},
+			{"pkts_in", "float", "both", 300, "packets/sec"},
+			{"pkts_out", "float", "both", 300, "packets/sec"},
+		},
+	},
+	{
+		Name: "process",
+		Metrics: []GangliaMetric{
+			{"proc_run", "uint32", "both", 950, ""},
+			{"proc_total", "uint32", "both", 950, ""},
+		},
+	},
+	{
+		Name: "system",
+		Metrics: []GangliaMetric{
+			{"boottime", "uint32", "zero", 1200, "s"},
+			{"sys_clock", "uint32", "zero", 1200, "s"},
+			{"machine_type", "string", "zero", 1200, ""},
+			{"os_name", "string", "zero", 1200, ""},
+			{"os_release", "string", "zero", 1200, ""},
+			{"mtu", "uint32", "both", 1200, ""},
+		},
+	},
+}
+
+type GangliaMetricConfig struct {
+	Type  string
+	Slope string
+	Tmax  int
+	Unit  string
+	Group string
+	Value string
+	Name  string
+}
+
+func GetCommonGangliaConfig(point lp.CCMessage) GangliaMetricConfig {
+	mname := GangliaMetricRename(point.Name())
+	if oldname, ok := point.GetMeta("oldname"); ok {
+		mname = GangliaMetricRename(oldname)
+	}
+	for _, group := range CommonGangliaMetrics {
+		for _, metric := range group.Metrics {
+			if metric.Name == mname {
+				valueStr := ""
+				value, ok := point.GetField("value")
+				if ok {
+					switch real := value.(type) {
+					case float64:
+						valueStr = fmt.Sprintf("%f", real)
+					case float32:
+						valueStr = fmt.Sprintf("%f", real)
+					case int64:
+						valueStr = fmt.Sprintf("%d", real)
+					case int32:
+						valueStr = fmt.Sprintf("%d", real)
+					case int:
+						valueStr = fmt.Sprintf("%d", real)
+					case uint64:
+						valueStr = fmt.Sprintf("%d", real)
+					case uint32:
+						valueStr = fmt.Sprintf("%d", real)
+					case uint:
+						valueStr = fmt.Sprintf("%d", real)
+					case string:
+						valueStr = real
+					default:
+					}
+				}
+				return GangliaMetricConfig{
+					Group: group.Name,
+					Type:  metric.Type,
+					Slope: metric.Slope,
+					Tmax:  metric.Tmax,
+					Unit:  metric.Unit,
+					Value: valueStr,
+					Name:  GangliaMetricRename(mname),
+				}
+			}
+		}
+	}
+	return GangliaMetricConfig{
+		Group: "",
+		Type:  "",
+		Slope: "",
+		Tmax:  0,
+		Unit:  "",
+		Value: "",
+		Name:  "",
+	}
+}
+
+func GetGangliaConfig(point lp.CCMessage) GangliaMetricConfig {
+	mname := GangliaMetricRename(point.Name())
+	if oldname, ok := point.GetMeta("oldname"); ok {
+		mname = GangliaMetricRename(oldname)
+	}
+	group := ""
+	if g, ok := point.GetMeta("group"); ok {
+		group = g
+	}
+	unit := ""
+	if u, ok := point.GetMeta("unit"); ok {
+		unit = u
+	}
+	valueType := "double"
+	valueStr := ""
+	value, ok := point.GetField("value")
+	if ok {
+		switch real := value.(type) {
+		case float64:
+			valueStr = fmt.Sprintf("%f", real)
+			valueType = "double"
+		case float32:
+			valueStr = fmt.Sprintf("%f", real)
+			valueType = "float"
+		case int64:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "int32"
+		case int32:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "int32"
+		case int:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "int32"
+		case uint64:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "uint32"
+		case uint32:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "uint32"
+		case uint:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "uint32"
+		case string:
+			valueStr = real
+			valueType = "string"
+		default:
+			valueType = "invalid"
+		}
+	}
+
+	return GangliaMetricConfig{
+		Group: group,
+		Type:  valueType,
+		Slope: DEFAULT_GANGLIA_METRIC_SLOPE,
+		Tmax:  DEFAULT_GANGLIA_METRIC_TMAX,
+		Unit:  unit,
+		Value: valueStr,
+		Name:  GangliaMetricRename(mname),
+	}
+}

sinks/gangliaSink.go

@@ -0,0 +1,147 @@
+package sinks
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	//	"time"
+	"os/exec"
+
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
+)
+
+const GMETRIC_EXEC = `gmetric`
+const GMETRIC_CONFIG = `/etc/ganglia/gmond.conf`
+
+type GangliaSinkConfig struct {
+	defaultSinkConfig
+	GmetricPath     string `json:"gmetric_path,omitempty"`
+	GmetricConfig   string `json:"gmetric_config,omitempty"`
+	AddGangliaGroup bool   `json:"add_ganglia_group,omitempty"`
+	AddTagsAsDesc   bool   `json:"add_tags_as_desc,omitempty"`
+	ClusterName     string `json:"cluster_name,omitempty"`
+	AddTypeToName   bool   `json:"add_type_to_name,omitempty"`
+	AddUnits        bool   `json:"add_units,omitempty"`
+}
+
+type GangliaSink struct {
+	sink
+	gmetric_path   string
+	gmetric_config string
+	config         GangliaSinkConfig
+}
+
+func (s *GangliaSink) Write(msg lp.CCMessage) error {
+	var err error = nil
+	//var tagsstr []string
+	var argstr []string
+
+	point, err := s.mp.ProcessMessage(msg)
+	if err == nil && point != nil {
+		// Get metric config (type, value, ... in suitable format)
+		conf := GetCommonGangliaConfig(point)
+		if len(conf.Type) == 0 {
+			conf = GetGangliaConfig(point)
+		}
+		if len(conf.Type) == 0 {
+			return fmt.Errorf("metric %q (Ganglia name %q) has no 'value' field", point.Name(), conf.Name)
+		}
+
+		if s.config.AddGangliaGroup {
+			argstr = append(argstr, fmt.Sprintf("--group=%s", conf.Group))
+		}
+		if s.config.AddUnits && len(conf.Unit) > 0 {
+			argstr = append(argstr, fmt.Sprintf("--units=%s", conf.Unit))
+		}
+
+		if len(s.config.ClusterName) > 0 {
+			argstr = append(argstr, fmt.Sprintf("--cluster=%s", s.config.ClusterName))
+		}
+		// if s.config.AddTagsAsDesc && len(tagsstr) > 0 {
+		// 	argstr = append(argstr, fmt.Sprintf("--desc=%q", strings.Join(tagsstr, ",")))
+		// }
+		if len(s.gmetric_config) > 0 {
+			argstr = append(argstr, fmt.Sprintf("--conf=%s", s.gmetric_config))
+		}
+		if s.config.AddTypeToName {
+			argstr = append(argstr, fmt.Sprintf("--name=%s", GangliaMetricName(point)))
+		} else {
+			argstr = append(argstr, fmt.Sprintf("--name=%s", conf.Name))
+		}
+		argstr = append(argstr, fmt.Sprintf("--slope=%s", conf.Slope))
+		argstr = append(argstr, fmt.Sprintf("--value=%s", conf.Value))
+		argstr = append(argstr, fmt.Sprintf("--type=%s", conf.Type))
+		argstr = append(argstr, fmt.Sprintf("--tmax=%d", conf.Tmax))
+
+		cclog.ComponentDebug(s.name, s.gmetric_path, strings.Join(argstr, " "))
+		command := exec.Command(s.gmetric_path, argstr...)
+		command.Wait()
+		_, err = command.Output()
+	}
+	return err
+}
+
+func (s *GangliaSink) Flush() error {
+	return nil
+}
+
+func (s *GangliaSink) Close() {
+}
+
+func NewGangliaSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(GangliaSink)
+	s.name = fmt.Sprintf("GangliaSink(%s)", name)
+	s.config.AddTagsAsDesc = false
+	s.config.AddGangliaGroup = false
+	if len(config) > 0 {
+		d := json.NewDecoder(bytes.NewReader(config))
+		d.DisallowUnknownFields()
+		if err := d.Decode(&s.config); err != nil {
+			cclog.ComponentError(s.name, "Error reading config:", err.Error())
+			return nil, err
+		}
+	}
+	s.gmetric_path = ""
+	s.gmetric_config = ""
+
+	p, err := mp.NewMessageProcessor()
+	if err != nil {
+		return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
+	}
+	s.mp = p
+
+	if len(s.config.GmetricPath) > 0 {
+		p, err := exec.LookPath(s.config.GmetricPath)
+		if err == nil {
+			s.gmetric_path = p
+		}
+	}
+	if len(s.gmetric_path) == 0 {
+		p, err := exec.LookPath(string(GMETRIC_EXEC))
+		if err == nil {
+			s.gmetric_path = p
+		}
+	}
+	if len(s.gmetric_path) == 0 {
+		return nil, errors.New("cannot find executable 'gmetric'")
+	}
+	if len(s.config.GmetricConfig) > 0 {
+		s.gmetric_config = s.config.GmetricConfig
+	}
+	if len(s.config.MessageProcessor) > 0 {
+		err = s.mp.FromConfigJSON(s.config.MessageProcessor)
+		if err != nil {
+			return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+		}
+	}
+	for _, k := range s.config.MetaAsTags {
+		s.mp.AddMoveMetaToTags("true", k, k)
+	}
+
+	return s, nil
+}

sinks/gangliaSink.md

@@ -0,0 +1,25 @@
+## `ganglia` sink
+
+The `ganglia` sink uses the `gmetric` tool of the [Ganglia Monitoring System](http://ganglia.info/) to submit the metrics
+
+### Configuration structure
+
+```json
+{
+  "<name>": {
+    "type": "ganglia",
+    "gmetric_path" : "/path/to/gmetric",
+    "add_ganglia_group" : true,
+    "process_messages" : {
+      "see" : "docs of message processor for valid fields"
+    },
+    "meta_as_tags" : []
+  }
+}
+```
+
+- `type`: makes the sink an `ganglia` sink
+- `gmetric_path`: Path to `gmetric` executable (optional). If not given, the sink searches in `$PATH` for `gmetric`.
+- `add_ganglia_group`: Add `--group=X` based on meta information to the `gmetric` call. Some old versions of `gmetric` do not support the `--group` option.
+- `process_messages`: Process messages with given rules before progressing or dropping, see [here](../pkg/messageProcessor/README.md) (optional)
+- `meta_as_tags`: print all meta information as tags in the output (deprecated, optional)

sinks/httpSink.go

@@ -0,0 +1,306 @@
+package sinks
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	lp "github.com/ClusterCockpit/cc-energy-manager/pkg/cc-message"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	mp "github.com/ClusterCockpit/cc-metric-collector/pkg/messageProcessor"
+	influx "github.com/influxdata/line-protocol/v2/lineprotocol"
+	"golang.org/x/exp/slices"
+)
+
+type HttpSinkConfig struct {
+	defaultSinkConfig
+
+	// The full URL of the endpoint
+	URL string `json:"url"`
+
+	// JSON web tokens for authentication (Using the *Bearer* scheme)
+	JWT string `json:"jwt,omitempty"`
+
+	// Basic authentication
+	Username     string `json:"username"`
+	Password     string `json:"password"`
+	useBasicAuth bool
+
+	// time limit for requests made by the http client
+	Timeout string `json:"timeout,omitempty"`
+	timeout time.Duration
+
+	// Maximum amount of time an idle (keep-alive) connection will remain idle before closing itself
+	// should be larger than the measurement interval to keep the connection open
+	IdleConnTimeout string `json:"idle_connection_timeout,omitempty"`
+	idleConnTimeout time.Duration
+
+	// Batch all writes arriving in during this duration
+	// (default '5s', batching can be disabled by setting it to 0)
+	FlushDelay string `json:"flush_delay,omitempty"`
+	flushDelay time.Duration
+
+	// Maximum number of retries to connect to the http server (default: 3)
+	MaxRetries int `json:"max_retries,omitempty"`
+
+	// Timestamp precision
+	Precision string `json:"precision,omitempty"`
+}
+
+type HttpSink struct {
+	sink
+	client *http.Client
+	// influx line protocol encoder
+	encoder influx.Encoder
+
+	// Flush() runs in another goroutine and accesses the influx line protocol encoder,
+	// so this encoderLock has to protect the encoder
+	encoderLock sync.Mutex
+
+	// timer to run Flush()
+	flushTimer *time.Timer
+	// Lock to assure that only one timer is running at a time
+	timerLock sync.Mutex
+
+	config HttpSinkConfig
+}
+
+// Write sends metric m as http message
+func (s *HttpSink) Write(msg lp.CCMessage) error {
+
+	// submit m only after applying processing/dropping rules
+	m, err := s.mp.ProcessMessage(msg)
+	if err == nil && m != nil {
+		// Lock for encoder usage
+		s.encoderLock.Lock()
+
+		err = EncoderAdd(&s.encoder, m)
+
+		// Unlock encoder usage
+		s.encoderLock.Unlock()
+
+		// Check that encoding worked
+		if err != nil {
+			return fmt.Errorf("encoding failed: %v", err)
+		}
+	}
+
+	if s.config.flushDelay == 0 {
+
+		// Directly flush if no flush delay is configured
+		return s.Flush()
+	} else if s.timerLock.TryLock() {
+
+		// Setup flush timer when flush delay is configured
+		// and no other timer is already running
+		if s.flushTimer != nil {
+
+			// Restarting existing flush timer
+			cclog.ComponentDebug(s.name, "Write(): Restarting flush timer")
+			s.flushTimer.Reset(s.config.flushDelay)
+		} else {
+
+			// Creating and starting flush timer
+			cclog.ComponentDebug(s.name, "Write(): Starting new flush timer")
+			s.flushTimer = time.AfterFunc(
+				s.config.flushDelay,
+				func() {
+					defer s.timerLock.Unlock()
+					cclog.ComponentDebug(s.name, "Starting flush triggered by flush timer")
+					if err := s.Flush(); err != nil {
+						cclog.ComponentError(s.name, "Flush triggered by flush timer: flush failed:", err)
+					}
+				})
+		}
+	}
+
+	return nil
+}
+
+// Flush sends all metrics stored in encoder to HTTP server
+func (s *HttpSink) Flush() error {
+
+	// Lock for encoder usage
+	// Own lock for as short as possible: the time it takes to clone the buffer.
+	s.encoderLock.Lock()
+
+	buf := slices.Clone(s.encoder.Bytes())
+	s.encoder.Reset()
+
+	// Unlock encoder usage
+	s.encoderLock.Unlock()
+
+	if len(buf) == 0 {
+		return nil
+	}
+
+	cclog.ComponentDebug(s.name, "Flush(): Flushing")
+
+	var res *http.Response
+	for i := 0; i < s.config.MaxRetries; i++ {
+		// Create new request to send buffer
+		req, err := http.NewRequest(http.MethodPost, s.config.URL, bytes.NewReader(buf))
+		if err != nil {
+			cclog.ComponentError(s.name, "Flush(): Failed to create HTTP request:", err)
+			return err
+		}
+
+		// Set authorization header
+		if len(s.config.JWT) != 0 {
+			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", s.config.JWT))
+		}
+
+		// Set basic authentication
+		if s.config.useBasicAuth {
+			req.SetBasicAuth(s.config.Username, s.config.Password)
+		}
+
+		// Do request
+		res, err = s.client.Do(req)
+		if err != nil {
+			cclog.ComponentError(s.name, "Flush(): transport/tcp error:", err)
+			// Wait between retries
+			time.Sleep(time.Duration(i+1) * (time.Second / 2))
+			continue
+		}
+
+		break
+	}
+
+	if res == nil {
+		return errors.New("flush failed due to repeated errors")
+	}
+
+	// Handle application errors
+	if res.StatusCode != http.StatusOK {
+		err := errors.New(res.Status)
+		cclog.ComponentError(s.name, "Flush(): Application error:", err)
+		return err
+	}
+
+	return nil
+}
+
+func (s *HttpSink) Close() {
+	cclog.ComponentDebug(s.name, "Closing HTTP connection")
+
+	// Stop existing timer and immediately flush
+	if s.flushTimer != nil {
+		if ok := s.flushTimer.Stop(); ok {
+			s.timerLock.Unlock()
+		}
+	}
+
+	// Flush
+	if err := s.Flush(); err != nil {
+		cclog.ComponentError(s.name, "Close(): Flush failed:", err)
+	}
+
+	s.client.CloseIdleConnections()
+}
+
+// NewHttpSink creates a new http sink
+func NewHttpSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(HttpSink)
+	// Set default values
+	s.name = fmt.Sprintf("HttpSink(%s)", name)
+	// should be larger than the measurement interval to keep the connection open
+	s.config.IdleConnTimeout = "120s"
+	s.config.Timeout = "5s"
+	s.config.FlushDelay = "5s"
+	s.config.MaxRetries = 3
+	s.config.Precision = "s"
+	cclog.ComponentDebug(s.name, "Init()")
+
+	// Read config
+	if len(config) > 0 {
+		d := json.NewDecoder(bytes.NewReader(config))
+		d.DisallowUnknownFields()
+		if err := d.Decode(&s.config); err != nil {
+			cclog.ComponentError(s.name, "Error reading config:", err.Error())
+			return nil, err
+		}
+	}
+	if len(s.config.URL) == 0 {
+		return nil, errors.New("`url` config option is required for HTTP sink")
+	}
+
+	// Check basic authentication config
+	if len(s.config.Username) > 0 || len(s.config.Password) > 0 {
+		s.config.useBasicAuth = true
+	}
+	if s.config.useBasicAuth && len(s.config.Username) == 0 {
+		return nil, errors.New("basic authentication requires username")
+	}
+	if s.config.useBasicAuth && len(s.config.Password) == 0 {
+		return nil, errors.New("basic authentication requires password")
+	}
+	p, err := mp.NewMessageProcessor()
+	if err != nil {
+		return nil, fmt.Errorf("initialization of message processor failed: %v", err.Error())
+	}
+	s.mp = p
+
+	if len(s.config.IdleConnTimeout) > 0 {
+		t, err := time.ParseDuration(s.config.IdleConnTimeout)
+		if err == nil {
+			cclog.ComponentDebug(s.name, "Init(): idleConnTimeout", t)
+			s.config.idleConnTimeout = t
+		}
+	}
+	if len(s.config.Timeout) > 0 {
+		t, err := time.ParseDuration(s.config.Timeout)
+		if err == nil {
+			s.config.timeout = t
+			cclog.ComponentDebug(s.name, "Init(): timeout", t)
+		}
+	}
+	if len(s.config.FlushDelay) > 0 {
+		t, err := time.ParseDuration(s.config.FlushDelay)
+		if err == nil {
+			s.config.flushDelay = t
+			cclog.ComponentDebug(s.name, "Init(): flushDelay", t)
+		}
+	}
+	if len(s.config.MessageProcessor) > 0 {
+		err = p.FromConfigJSON(s.config.MessageProcessor)
+		if err != nil {
+			return nil, fmt.Errorf("failed parsing JSON for message processor: %v", err.Error())
+		}
+	}
+	for _, k := range s.config.MetaAsTags {
+		s.mp.AddMoveMetaToTags("true", k, k)
+	}
+
+	precision := influx.Second
+	if len(s.config.Precision) > 0 {
+		switch s.config.Precision {
+		case "s":
+			precision = influx.Second
+		case "ms":
+			precision = influx.Millisecond
+		case "us":
+			precision = influx.Microsecond
+		case "ns":
+			precision = influx.Nanosecond
+		}
+	}
+
+	// Create http client
+	s.client = &http.Client{
+		Transport: &http.Transport{
+			MaxIdleConns:    1, // We will only ever talk to one host.
+			IdleConnTimeout: s.config.idleConnTimeout,
+		},
+		Timeout: s.config.timeout,
+	}
+
+	// Configure influx line protocol encoder
+	s.encoder.SetPrecision(precision)
+
+	return s, nil
+}

sinks/httpSink.md

@@ -0,0 +1,44 @@
+## `http` sink
+
+The `http` sink uses POST requests to a HTTP server to submit the metrics in the InfluxDB line-protocol format. It uses JSON web tokens for authentification. The sink creates batches of metrics before sending, to reduce the HTTP traffic.
+
+### Configuration structure
+
+```json
+{
+  "<name>": {
+    "type": "http",
+    "url" : "https://my-monitoring.example.com:1234/api/write",
+    "jwt" : "blabla.blabla.blabla",
+    "username": "myUser",
+    "password": "myPW",
+    "timeout": "5s",
+    "idle_connection_timeout" : "5s",
+    "flush_delay": "2s",
+    "batch_size": 1000,
+    "precision": "s",
+    "process_messages" : {
+      "see" : "docs of message processor for valid fields"
+    },
+    "meta_as_tags" : []
+  }
+}
+```
+
+- `type`: makes the sink an `http` sink
+- `url`: The full URL of the endpoint
+- `jwt`: JSON web tokens for authentication (Using the *Bearer* scheme)
+- `username`: username for basic authentication
+- `password`: password for basic authentication
+- `timeout`: General timeout for the HTTP client (default '5s')
+- `max_retries`: Maximum number of retries to connect to the http server
+- `idle_connection_timeout`: Timeout for idle connections (default '120s'). Should be larger than the measurement interval to keep the connection open
+- `flush_delay`: Batch all writes arriving in during this duration (default '1s', batching can be disabled by setting it to 0)
+- `batch_size`: Maximal batch size. If `batch_size` is reached before the end of `flush_delay`, the metrics are sent without further delay
+- `precision`: Precision of the timestamp. Valid values are 's', 'ms', 'us' and 'ns'. (default is 's')
+- `process_messages`: Process messages with given rules before progressing or dropping, see [here](../pkg/messageProcessor/README.md) (optional)
+- `meta_as_tags`: print all meta information as tags in the output (deprecated, optional)
+
+### Using `http` sink for communication with cc-metric-store
+
+The cc-metric-store only accepts metrics with a timestamp precision in seconds, so it is required to use `"precision": "s"`.