Merge branch 'develop' into main

* Update configuration.md

Add an additional receiver to have better alignment of components

* Change default GpfsCollector command to `mmpmon` (#53)

* Set default cmd to 'mmpmon'

* Reuse looked up path

* Cast const to string

* Just download LIKWID to get the headers (#54)

* Just download LIKWID to get the headers

* Remove perl-Data-Dumper from BuildRequires, only required by LIKWID build

* Add HttpReceiver as counterpart to the HttpSink (#49)

* Use GBytes as unit for large memory numbers

* Make maxForward configurable, save old name in meta in rename metrics and make the hostname tag key configurable

* Single release action (#55)

Building all RPMs and releasing in a single workflow

* Makefile target to build binary-only Debian packages (#61)

* Add 'install' and 'DEB' make targets to build binary-only Debian packages

* Add control file for DEB builds

* Use a single line for bash loop in make clean

* Add config options for retry intervals of InfluxDB clients (#59)

* Refactoring of LikwidCollector and metric units (#62)

* Reduce complexity of LikwidCollector and allow metric units

* Add unit to LikwidCollector docu and fix some typos

* Make library path configurable

* Use old metric name in Ganglia if rename has happened in the router (#60)

* Use old metric name if rename has happened in the router

* Also check for Ganglia renames for the oldname

* Derived metrics (#57)

* Add time-based derivatived (e.g. bandwidth) to some collectors

* Add documentation

* Add comments

* Fix: Only compute rates with a valid previous state

* Only compute rates with a valid previous state

* Define const values for net/dev fields

* Set default config values

* Add comments

* Refactor: Consolidate data structures

* Refactor: Consolidate data structures

* Refactor: Avoid struct deep copy

* Refactor: Avoid redundant tag maps

* Refactor: Use int64 type for absolut values

Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com>

* Simplified iota usage

* Move unit tag to meta data tags

* Derived metrics (#65)

* Add time-based derivatived (e.g. bandwidth) to some collectors

* Add documentation

* Add comments

* Fix: Only compute rates with a valid previous state

* Only compute rates with a valid previous state

* Define const values for net/dev fields

* Set default config values

* Add comments

* Refactor: Consolidate data structures

* Refactor: Consolidate data structures

* Refactor: Avoid struct deep copy

* Refactor: Avoid redundant tag maps

* Refactor: Use int64 type for absolut values

* Update LustreCollector

Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com>

* Meta to tags list and map for sinks (#63)

* Change ccMetric->Influx functions

* Use a meta_as_tags string list in config but create a lookup map afterwards

* Add meta as tag logic to sampleSink

* Fix staticcheck warnings (#66)

Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com>

.github/workflows/Release.yml

@@ -0,0 +1,184 @@
+# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+
+# Workflow name
+name: Release
+
+# Run on tag push
+on:
+ push:
+  tags:
+  - '**'
+
+jobs:
+
+  #
+  # Build on AlmaLinux 8.5 using go-toolset
+  #
+  AlmaLinux-RPM-build:
+    runs-on: ubuntu-latest
+    # See: https://hub.docker.com/_/almalinux
+    container: almalinux:8.5
+    # The job outputs link to the outputs of the 'rpmrename' step
+    # Only job outputs can be used in child jobs
+    outputs:
+      rpm : ${{steps.rpmrename.outputs.RPM}}
+      srpm : ${{steps.rpmrename.outputs.SRPM}}
+    steps:
+
+    # Use dnf to install development packages
+    - name: Install development packages
+      run: dnf --assumeyes group install "Development Tools" "RPM Development Tools"
+
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    # Use dnf to install build dependencies
+    - name: Install build dependencies
+      run: dnf --assumeyes builddep scripts/cc-metric-collector.spec
+
+    - name: RPM build MetricCollector
+      id: rpmbuild
+      run: make RPM
+    
+    # AlmaLinux 8.5 is a derivate of RedHat Enterprise Linux 8 (UBI8),
+    # so the created RPM both contain the substring 'el8' in the RPM file names
+    # This step replaces the substring 'el8' to 'alma85'. It uses the move operation
+    # because it is unclear whether the default AlmaLinux 8.5 container contains the 
+    # 'rename' command. This way we also get the new names for output.
+    - name: Rename RPMs (s/el8/alma85/)
+      id: rpmrename
+      run: |
+        OLD_RPM="${{steps.rpmbuild.outputs.RPM}}"
+        OLD_SRPM="${{steps.rpmbuild.outputs.SRPM}}"
+        NEW_RPM="${OLD_RPM/el8/alma85}"
+        NEW_SRPM=${OLD_SRPM/el8/alma85}
+        mv "${OLD_RPM}" "${NEW_RPM}"
+        mv "${OLD_SRPM}" "${NEW_SRPM}"
+        echo "::set-output name=SRPM::${NEW_SRPM}"
+        echo "::set-output name=RPM::${NEW_RPM}"
+
+    # See: https://github.com/actions/upload-artifact
+    - name: Save RPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-collector RPM for AlmaLinux 8.5
+        path: ${{ steps.rpmrename.outputs.RPM }}
+    - name: Save SRPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-collector SRPM for AlmaLinux 8.5
+        path: ${{ steps.rpmrename.outputs.SRPM }}
+
+  #
+  # Build on UBI 8 using go-toolset
+  #
+  UBI-8-RPM-build:
+    runs-on: ubuntu-latest
+    # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti
+    container: registry.access.redhat.com/ubi8/ubi:8.5-226.1645809065
+    # The job outputs link to the outputs of the 'rpmbuild' step
+    outputs:
+      rpm : ${{steps.rpmbuild.outputs.RPM}}
+      srpm : ${{steps.rpmbuild.outputs.SRPM}}
+    steps:
+
+    # Use dnf to install development packages
+    - name: Install development packages
+      run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git
+
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    # Use dnf to install build dependencies
+    - name: Install build dependencies
+      run: dnf --assumeyes --disableplugin=subscription-manager builddep scripts/cc-metric-collector.spec
+
+    - name: RPM build MetricCollector
+      id: rpmbuild
+      run: make RPM
+
+    # See: https://github.com/actions/upload-artifact
+    - name: Save RPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-collector RPM for UBI 8
+        path: ${{ steps.rpmbuild.outputs.RPM }}
+    - name: Save SRPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-collector SRPM for UBI 8
+        path: ${{ steps.rpmbuild.outputs.SRPM }}
+
+  #
+  # Create release with fresh RPMs
+  #
+  Release:
+    runs-on: ubuntu-latest
+    # We need the RPMs, so add dependency
+    needs: [AlmaLinux-RPM-build, UBI-8-RPM-build]
+
+    steps:
+    # See: https://github.com/actions/download-artifact
+    - name: Download AlmaLinux 8.5 RPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-metric-collector RPM for AlmaLinux 8.5
+    - name: Download AlmaLinux 8.5 SRPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-metric-collector SRPM for AlmaLinux 8.5
+    
+    - name: Download UBI 8 RPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-metric-collector RPM for UBI 8
+    - name: Download UBI 8 SRPM
+      uses: actions/download-artifact@v2
+      with:
+        name: cc-metric-collector SRPM for UBI 8
+
+    # The download actions do not publish the name of the downloaded file,
+    # so we re-use the job outputs of the parent jobs. The files are all
+    # downloaded to the current folder.
+    # The gh-release action afterwards does not accept file lists but all
+    # files have to be listed at 'files'. The step creates one output per
+    # RPM package (2 per distro)
+    - name: Set RPM variables
+      id: files
+      run: |
+        ALMA_85_RPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.rpm}}")
+        ALMA_85_SRPM=$(basename "${{ needs.AlmaLinux-RPM-build.outputs.srpm}}")
+        UBI_8_RPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.rpm}}")
+        UBI_8_SRPM=$(basename "${{ needs.UBI-8-RPM-build.outputs.srpm}}")
+        echo "ALMA_85_RPM::${ALMA_85_RPM}"
+        echo "ALMA_85_SRPM::${ALMA_85_SRPM}"
+        echo "UBI_8_RPM::${UBI_8_RPM}"
+        echo "UBI_8_SRPM::${UBI_8_SRPM}"
+        echo "::set-output name=ALMA_85_RPM::${ALMA_85_RPM}"
+        echo "::set-output name=ALMA_85_SRPM::${ALMA_85_SRPM}"
+        echo "::set-output name=UBI_8_RPM::${UBI_8_RPM}"
+        echo "::set-output name=UBI_8_SRPM::${UBI_8_SRPM}"
+
+    # See: https://github.com/softprops/action-gh-release
+    - name: Release
+      uses: softprops/action-gh-release@v1
+      if: startsWith(github.ref, 'refs/tags/')
+      with:
+        name: cc-metric-collector-${{github.ref_name}}
+        files: |
+         ${{ steps.files.outputs.ALMA_85_RPM }}
+         ${{ steps.files.outputs.ALMA_85_SRPM }}
+         ${{ steps.files.outputs.UBI_8_RPM }}
+         ${{ steps.files.outputs.UBI_8_SRPM }}

.github/workflows/rpmbuild.yml

@@ -1,58 +0,0 @@
-name: Run RPM Build
-on:
- push:
-  tags:
-  - '**'
-
-jobs:
-  build-alma-8_5:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: TomTheBear/rpmbuild@alma8.5
-      id: rpm
-      name: Build RPM package on AlmaLinux 8.5
-      with:
-        spec_file: "./scripts/cc-metric-collector.spec"
-    - name: Save RPM as artifact
-      uses: actions/upload-artifact@v1.0.0
-      with:
-        name: cc-metric-collector RPM AlmaLinux 8.5
-        path: ${{ steps.rpm.outputs.rpm_path }}
-    - name: Save SRPM as artifact
-      uses: actions/upload-artifact@v1.0.0
-      with:
-        name: cc-metric-collector SRPM AlmaLinux 8.5
-        path: ${{ steps.rpm.outputs.source_rpm_path }}
-    - name: Release
-      uses: softprops/action-gh-release@v1
-      with:
-        name: cc-metric-collector-${{github.ref_name}}
-        files: |
-         ${{ steps.rpm.outputs.source_rpm_path }}
-         ${{ steps.rpm.outputs.rpm_path }}
-  build-rhel-ubi8:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: TomTheBear/rpmbuild@rh-ubi8
-      id: rpm
-      name: Build RPM package on Red Hat Universal Base Image 8
-      with:
-        spec_file: "./scripts/cc-metric-collector.spec"
-    - name: Save RPM as artifact
-      uses: actions/upload-artifact@v1.0.0
-      with:
-        name: cc-metric-collector RPM Red Hat Universal Base Image 8
-        path: ${{ steps.rpm.outputs.rpm_path }}
-    - name: Save SRPM as artifact
-      uses: actions/upload-artifact@v1.0.0
-      with:
-        name: cc-metric-collector SRPM Red Hat Universal Base Image 8
-        path: ${{ steps.rpm.outputs.source_rpm_path }}
-    - name: Release
-      uses: softprops/action-gh-release@v1
-      with:
-        files: |
-         ${{ steps.rpm.outputs.source_rpm_path }}
-         ${{ steps.rpm.outputs.rpm_path }}

.github/workflows/runonce.yml

@@ -1,46 +1,68 @@
+# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+
+# Workflow name
 name: Run Test
+
+# Run on event push
 on: push
 
 jobs:
+  #
+  # Job build-1-17
+  # Build on latest Ubuntu using golang version 1.17 
+  #
   build-1-17:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    # See: https://github.com/marketplace/actions/checkout
+    # Checkout git repository and submodules
+    - name: Checkout
+      uses: actions/checkout@v2
       with:
         submodules: recursive
 
     # See: https://github.com/marketplace/actions/setup-go-environment
     - name: Setup Golang
-      uses: actions/setup-go@v2.1.5
+      uses: actions/setup-go@v2
       with:
-        go-version: '^1.17.6'
+        go-version: '^1.17.7'
 
+    # Install libganglia
     - name: Setup Ganglia
       run: sudo apt install ganglia-monitor libganglia1
 
     - name: Build MetricCollector
       run: make
 
-    - name: Run MetricCollector
+    - name: Run MetricCollector once
       run: ./cc-metric-collector --once --config .github/ci-config.json
+
+  #
+  # Job build-1-16
+  # Build on latest Ubuntu using golang version 1.16
+  #
   build-1-16:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    # See: https://github.com/marketplace/actions/checkout
+    # Checkout git repository and submodules
+    - name: Checkout
+      uses: actions/checkout@v2
       with:
         submodules: recursive
 
     # See: https://github.com/marketplace/actions/setup-go-environment
     - name: Setup Golang
-      uses: actions/setup-go@v2.1.5
+      uses: actions/setup-go@v2
       with:
         go-version: '^1.16.7' # The version AlmaLinux 8.5 uses
 
+    # Install libganglia
     - name: Setup Ganglia
       run: sudo apt install ganglia-monitor libganglia1
 
     - name: Build MetricCollector
       run: make
 
-    - name: Run MetricCollectorlibganglia1
+    - name: Run MetricCollector once
       run: ./cc-metric-collector --once --config .github/ci-config.json

Makefile

@@ -1,5 +1,5 @@
 APP = cc-metric-collector
-GOSRC_APP        := metric-collector.go
+GOSRC_APP        := cc-metric-collector.go
 GOSRC_COLLECTORS := $(wildcard collectors/*.go)
 GOSRC_SINKS      := $(wildcard sinks/*.go)
 GOSRC_RECEIVERS  := $(wildcard receivers/*.go)
@@ -15,19 +15,38 @@ COMPONENT_DIRS   := collectors \
 			internal/ccTopology \
 			internal/multiChanTicker
 
+BINDIR = bin
+
 
 .PHONY: all
 all: $(APP)
 
 $(APP): $(GOSRC)
 	make -C collectors
-	make -C sinks
 	go get
 	go build -o $(APP) $(GOSRC_APP)
 
+install: $(APP)
+	@WORKSPACE=$(PREFIX)
+	@if [ -z "$${WORKSPACE}" ]; then exit 1; fi
+	@mkdir --parents --verbose $${WORKSPACE}/usr/$(BINDIR)
+	@install -Dpm 755 $(APP) $${WORKSPACE}/usr/$(BINDIR)/$(APP)
+	@mkdir --parents --verbose $${WORKSPACE}/etc/cc-metric-collector $${WORKSPACE}/etc/default $${WORKSPACE}/etc/systemd/system $${WORKSPACE}/etc/init.d
+	@install -Dpm 600 config.json $${WORKSPACE}/etc/cc-metric-collector/cc-metric-collector.json
+	@sed -i -e s+"\"./"+"\"/etc/cc-metric-collector/"+g $${WORKSPACE}/etc/cc-metric-collector/cc-metric-collector.json
+	@install -Dpm 600 sinks.json $${WORKSPACE}/etc/cc-metric-collector/sinks.json
+	@install -Dpm 600 collectors.json $${WORKSPACE}/etc/cc-metric-collector/collectors.json
+	@install -Dpm 600 router.json $${WORKSPACE}/etc/cc-metric-collector/router.json
+	@install -Dpm 600 receivers.json $${WORKSPACE}/etc/cc-metric-collector/receivers.json
+	@install -Dpm 600 scripts/cc-metric-collector.config $${WORKSPACE}/etc/default/cc-metric-collector
+	@install -Dpm 644 scripts/cc-metric-collector.service $${WORKSPACE}/etc/systemd/system/cc-metric-collector.service
+	@install -Dpm 644 scripts/cc-metric-collector.init $${WORKSPACE}/etc/init.d/cc-metric-collector
+
+
 .PHONY: clean
+.ONESHELL:
 clean:
-	@for COMP in $(COMPONENT_DIRS); do if [ -e $$COMP/Makefile ]; then make -C $$COMP clean; fi; done
+	@for COMP in $(COMPONENT_DIRS); do if [ -e $$COMP/Makefile ]; then make -C $$COMP clean;  fi; done
 	rm -f $(APP)
 
 .PHONY: fmt
@@ -51,3 +70,58 @@ vet:
 staticcheck:
 	go install honnef.co/go/tools/cmd/staticcheck@latest
 	$$(go env GOPATH)/bin/staticcheck ./...
+
+.ONESHELL:
+.PHONY: RPM
+RPM: scripts/cc-metric-collector.spec
+	@WORKSPACE="$${PWD}"
+	@SPECFILE="$${WORKSPACE}/scripts/cc-metric-collector.spec"
+	# Setup RPM build tree
+	@eval $$(rpm --eval "ARCH='%{_arch}' RPMDIR='%{_rpmdir}' SOURCEDIR='%{_sourcedir}' SPECDIR='%{_specdir}' SRPMDIR='%{_srcrpmdir}' BUILDDIR='%{_builddir}'")
+	@mkdir --parents --verbose "$${RPMDIR}" "$${SOURCEDIR}" "$${SPECDIR}" "$${SRPMDIR}" "$${BUILDDIR}"
+	# Create source tarball
+	@COMMITISH="HEAD"
+	@VERS=$$(git describe --tags $${COMMITISH})
+	@VERS=$${VERS#v}
+	@VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g)
+	@eval $$(rpmspec --query --queryformat "NAME='%{name}' VERSION='%{version}' RELEASE='%{release}' NVR='%{NVR}' NVRA='%{NVRA}'" --define="VERS $${VERS}" "$${SPECFILE}")
+	@PREFIX="$${NAME}-$${VERSION}"
+	@FORMAT="tar.gz"
+	@SRCFILE="$${SOURCEDIR}/$${PREFIX}.$${FORMAT}"
+	@git archive --verbose --format "$${FORMAT}" --prefix="$${PREFIX}/" --output="$${SRCFILE}" $${COMMITISH}
+	# Build RPM and SRPM
+	@rpmbuild -ba --define="VERS $${VERS}" --rmsource --clean "$${SPECFILE}"
+	# Report RPMs and SRPMs when in GitHub Workflow
+	@if [[ "$${GITHUB_ACTIONS}" == true ]]; then
+	@     RPMFILE="$${RPMDIR}/$${ARCH}/$${NVRA}.rpm"
+	@     SRPMFILE="$${SRPMDIR}/$${NVR}.src.rpm"
+	@     echo "RPM: $${RPMFILE}"
+	@     echo "SRPM: $${SRPMFILE}"
+	@     echo "::set-output name=SRPM::$${SRPMFILE}"
+	@     echo "::set-output name=RPM::$${RPMFILE}"
+	@fi
+
+.PHONY: DEB
+DEB: scripts/cc-metric-collector.deb.control $(APP)
+	@BASEDIR=$${PWD}
+	@WORKSPACE=$${PWD}/.dpkgbuild
+	@DEBIANDIR=$${WORKSPACE}/debian
+	@DEBIANBINDIR=$${WORKSPACE}/DEBIAN
+	@mkdir --parents --verbose $$WORKSPACE $$DEBIANBINDIR
+	#@mkdir --parents --verbose $$DEBIANDIR
+	@CONTROLFILE="$${BASEDIR}/scripts/cc-metric-collector.deb.control"
+	@COMMITISH="HEAD"
+	@VERS=$$(git describe --tags --abbrev=0 $${COMMITISH})
+	@VERS=$${VERS#v}
+	@VERS=$$(echo $$VERS | sed -e s+'-'+'_'+g)
+	@ARCH=$$(uname -m)
+	@ARCH=$$(echo $$ARCH | sed -e s+'_'+'-'+g)
+	@PREFIX="$${NAME}-$${VERSION}_$${ARCH}"
+	@SIZE_BYTES=$$(du -bcs --exclude=.dpkgbuild "$$WORKSPACE"/ | awk '{print $$1}' | head -1 | sed -e 's/^0\+//')
+	@SIZE="$$(awk -v size="$$SIZE_BYTES" 'BEGIN {print (size/1024)+1}' | awk '{print int($$0)}')"
+	#@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANDIR}/control
+	@sed -e s+"{VERSION}"+"$$VERS"+g -e s+"{INSTALLED_SIZE}"+"$$SIZE"+g -e s+"{ARCH}"+"$$ARCH"+g $$CONTROLFILE > $${DEBIANBINDIR}/control
+	@make PREFIX=$${WORKSPACE} install
+	@DEB_FILE="cc-metric-collector_$${VERS}_$${ARCH}.deb"
+	@dpkg-deb -b $${WORKSPACE} "$$DEB_FILE"
+	@rm -r "$${WORKSPACE}"

README.md

@@ -56,7 +56,41 @@ Usage of metric-collector:
   -once
     	Run all collectors only once
 ```
+# Scenarios
 
+The metric collector was designed with flexibility in mind, so it can be used in many scenarios. Here are a few:
+
+```mermaid
+flowchart TD
+  subgraph a ["Cluster A"]
+  nodeA[NodeA with CC collector]
+  nodeB[NodeB with CC collector]
+  nodeC[NodeC with CC collector]
+  end
+  a --> db[(Database)]
+  db <--> ccweb("Webfrontend")
+```
+
+``` mermaid
+flowchart TD
+  subgraph a [ClusterA]
+  direction LR
+  nodeA[NodeA with CC collector]
+  nodeB[NodeB with CC collector]
+  nodeC[NodeC with CC collector]
+  end
+  subgraph b [ClusterB]
+  direction LR
+  nodeD[NodeD with CC collector]
+  nodeE[NodeE with CC collector]
+  nodeF[NodeF with CC collector]
+  end
+  a --> ccrecv{"CC collector as receiver"}
+  b --> ccrecv
+  ccrecv --> db[("Database1")]
+  ccrecv -.-> db2[("Database2")]
+  db <-.-> ccweb("Webfrontend")
+```
 
 # Contributing
 The ClusterCockpit ecosystem is designed to be used by different HPC computing centers. Since configurations and setups differ between the centers, the centers likely have to put some work into the cc-metric-collector to gather all desired metrics.

collectors.json

@@ -12,6 +12,12 @@
             "proc_total"
         ]
     },
+    "netstat": {
+        "include_devices": [
+            "enp5s0"
+        ],
+        "send_derived_values": true
+    },
     "numastats": {},
     "nvidia": {},
     "tempstat": {

collectors/Makefile

@@ -1,82 +1,25 @@
-# Use central installation
-CENTRAL_INSTALL = false
-# How to access hardware performance counters through LIKWID.
-# Recommended is 'direct' mode
-ACCESSMODE = direct
 
-#######################################################################
-# if CENTRAL_INSTALL == true
-#######################################################################
-# Path to central installation (if CENTRAL_INSTALL=true)
-LIKWID_BASE=/apps/likwid/5.2.1
-# LIKWID version (should be same major version as central installation, 5.2.x)
+all: likwid
+
+
+# LIKWID version
 LIKWID_VERSION = 5.2.1
 
-#######################################################################
-# if CENTRAL_INSTALL == false and ACCESSMODE == accessdaemon
-#######################################################################
-# Where to install the accessdaemon
-DAEMON_INSTALLDIR = /usr/local
-# Which user to use for the accessdaemon
-DAEMON_USER = root
-# Which group to use for the accessdaemon
-DAEMON_GROUP = root
+.ONESHELL:
+.PHONY: likwid
+likwid:
+	INSTALL_FOLDER="$${PWD}/likwid"
+	BUILD_FOLDER="$${PWD}/likwidbuild"
+	if [ -d $${INSTALL_FOLDER} ]; then rm -r $${INSTALL_FOLDER}; fi
+	mkdir --parents --verbose  $${INSTALL_FOLDER} $${BUILD_FOLDER}
+	wget -P "$${BUILD_FOLDER}" ftp://ftp.rrze.uni-erlangen.de/mirrors/likwid/likwid-$(LIKWID_VERSION).tar.gz
+	tar -C $${BUILD_FOLDER} -xf $${BUILD_FOLDER}/likwid-$(LIKWID_VERSION).tar.gz
+	install -Dpm 0644 $${BUILD_FOLDER}/likwid-$(LIKWID_VERSION)/src/includes/likwid*.h $${INSTALL_FOLDER}/
+	install -Dpm 0644 $${BUILD_FOLDER}/likwid-$(LIKWID_VERSION)/src/includes/bstrlib.h $${INSTALL_FOLDER}/
+	rm -r $${BUILD_FOLDER}
 
 
-
-#################################################
-# No need to change anything below this line
-#################################################
-INSTALL_FOLDER = ./likwid
-BUILD_FOLDER = ./likwid/build
-
-ifneq ($(strip $(CENTRAL_INSTALL)),true)
-LIKWID_BASE := $(shell pwd)/$(INSTALL_FOLDER)
-DAEMON_BASE := $(LIKWID_BASE)
-GROUPS_BASE := $(LIKWID_BASE)/groups
-all: $(INSTALL_FOLDER)/liblikwid.a cleanup
-else
-DAEMON_BASE= $(LIKWID_BASE)/sbin
-all: $(INSTALL_FOLDER)/liblikwid.a cleanup
-endif
-
-
-
-$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION).tar.gz: $(BUILD_FOLDER)
-	wget -P $(BUILD_FOLDER) ftp://ftp.rrze.uni-erlangen.de/mirrors/likwid/likwid-$(LIKWID_VERSION).tar.gz
-
-$(BUILD_FOLDER):
-	mkdir -p $(BUILD_FOLDER)
-
-$(INSTALL_FOLDER):
-	mkdir -p $(INSTALL_FOLDER)
-
-$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION): $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION).tar.gz
-	tar -C $(BUILD_FOLDER) -xf $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION).tar.gz
-
-$(INSTALL_FOLDER)/liblikwid.a: $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION) $(INSTALL_FOLDER)
-	sed -i -e s+"PREFIX ?= .*"+"PREFIX = $(LIKWID_BASE)"+g \
-		-e s+"SHARED_LIBRARY = .*"+"SHARED_LIBRARY = false"+g \
-		-e s+"ACCESSMODE = .*"+"ACCESSMODE = $(ACCESSMODE)"+g \
-		-e s+"INSTALLED_ACCESSDAEMON = .*"+"INSTALLED_ACCESSDAEMON = $(DAEMON_INSTALLDIR)/likwid-accessD"+g \
-		$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/config.mk
-	cd $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION) && make
-	cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/liblikwid.a $(INSTALL_FOLDER)
-	cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/ext/hwloc/liblikwid-hwloc.a $(INSTALL_FOLDER)
-	cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/src/includes/likwid*.h $(INSTALL_FOLDER)
-	cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/src/includes/bstrlib.h $(INSTALL_FOLDER)
-
-$(DAEMON_INSTALLDIR)/likwid-accessD: $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/likwid-accessD
-	sudo -u $(DAEMON_USER) -g $(DAEMON_GROUP) install -m 4775 $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/likwid-accessD $(DAEMON_INSTALLDIR)/likwid-accessD
-
-prepare_collector: likwidMetric.go
-	cp likwidMetric.go likwidMetric.go.orig
-	sed -i -e s+"const GROUPPATH =.*"+"const GROUPPATH = \`$(GROUPS_BASE)\`"+g likwidMetric.go
-
-cleanup:
-	rm -rf $(BUILD_FOLDER)
-
-clean: cleanup
+clean:
 	rm -rf likwid
 
 .PHONY: clean

collectors/README.md

@@ -37,6 +37,8 @@ In contrast to the configuration files for sinks and receivers, the collectors c
 * [`cpufreq_cpuinfo`](./cpufreqCpuinfoMetric.md)
 * [`numastat`](./numastatMetric.md)
 * [`gpfs`](./gpfsMetric.md)
+* [`beegfs_meta`](./beegfsmetaMetric.md)
+* [`beegfs_storage`](./beegfsstorageMetric.md)
 
 ## Todos
 

collectors/beegfsmetaMetric.go

@@ -0,0 +1,229 @@
+package collectors
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"os/user"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+)
+
+const DEFAULT_BEEGFS_CMD = "beegfs-ctl"
+
+// Struct for the collector-specific JSON config
+type BeegfsMetaCollectorConfig struct {
+	Beegfs            string   `json:"beegfs_path"`
+	ExcludeMetrics    []string `json:"exclude_metrics,omitempty"`
+	ExcludeFilesystem []string `json:"exclude_filesystem"`
+}
+
+type BeegfsMetaCollector struct {
+	metricCollector
+	tags    map[string]string
+	matches map[string]string
+	config  BeegfsMetaCollectorConfig
+	skipFS  map[string]struct{}
+}
+
+func (m *BeegfsMetaCollector) Init(config json.RawMessage) error {
+	// Check if already initialized
+	if m.init {
+		return nil
+	}
+	// Metrics
+	var nodeMdstat_array = [39]string{
+		"sum", "ack", "close", "entInf",
+		"fndOwn", "mkdir", "create", "rddir",
+		"refrEn", "mdsInf", "rmdir", "rmLnk",
+		"mvDirIns", "mvFiIns", "open", "ren",
+		"sChDrct", "sAttr", "sDirPat", "stat",
+		"statfs", "trunc", "symlnk", "unlnk",
+		"lookLI", "statLI", "revalLI", "openLI",
+		"createLI", "hardlnk", "flckAp", "flckEn",
+		"flckRg", "dirparent", "listXA", "getXA",
+		"rmXA", "setXA", "mirror"}
+
+	m.name = "BeegfsMetaCollector"
+	m.setup()
+	// Set default beegfs-ctl binary
+
+	m.config.Beegfs = DEFAULT_BEEGFS_CMD
+
+	// Read JSON configuration
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &m.config)
+		if err != nil {
+			return err
+		}
+	}
+
+	//create map with possible variables
+	m.matches = make(map[string]string)
+	for _, value := range nodeMdstat_array {
+		_, skip := stringArrayContains(m.config.ExcludeMetrics, value)
+		if skip {
+			m.matches["other"] = "0"
+		} else {
+			m.matches["beegfs_cmeta_"+value] = "0"
+		}
+	}
+
+	m.meta = map[string]string{
+		"source": m.name,
+		"group":  "BeegfsMeta",
+	}
+	m.tags = map[string]string{
+		"type":       "node",
+		"filesystem": "",
+	}
+	m.skipFS = make(map[string]struct{})
+	for _, fs := range m.config.ExcludeFilesystem {
+		m.skipFS[fs] = struct{}{}
+	}
+
+	// Beegfs file system statistics can only be queried by user root
+	user, err := user.Current()
+	if err != nil {
+		return fmt.Errorf("BeegfsMetaCollector.Init(): Failed to get current user: %v", err)
+	}
+	if user.Uid != "0" {
+		return fmt.Errorf("BeegfsMetaCollector.Init(): BeeGFS file system statistics can only be queried by user root")
+	}
+
+	// Check if beegfs-ctl is in executable search path
+	_, err = exec.LookPath(m.config.Beegfs)
+	if err != nil {
+		return fmt.Errorf("BeegfsMetaCollector.Init(): Failed to find beegfs-ctl binary '%s': %v", m.config.Beegfs, err)
+	}
+	m.init = true
+	return nil
+}
+
+func (m *BeegfsMetaCollector) Read(interval time.Duration, output chan lp.CCMetric) {
+	if !m.init {
+		return
+	}
+	//get mounpoint
+	buffer, _ := ioutil.ReadFile(string("/proc/mounts"))
+	mounts := strings.Split(string(buffer), "\n")
+	var mountpoints []string
+	for _, line := range mounts {
+		if len(line) == 0 {
+			continue
+		}
+		f := strings.Fields(line)
+		if strings.Contains(f[0], "beegfs_ondemand") {
+			// Skip excluded filesystems
+			if _, skip := m.skipFS[f[1]]; skip {
+				continue
+			}
+			mountpoints = append(mountpoints, f[1])
+		}
+	}
+
+	if len(mountpoints) == 0 {
+		return
+	}
+
+	for _, mountpoint := range mountpoints {
+		m.tags["filesystem"] = mountpoint
+
+		// bwwgfs-ctl:
+		// --clientstats: Show client IO statistics.
+		// --nodetype=meta: The node type to query (meta, storage).
+		// --interval:
+		// --mount=/mnt/beeond/: Which mount point
+		//cmd := exec.Command(m.config.Beegfs, "/root/mc/test.txt")
+		mountoption := "--mount=" + mountpoint
+		cmd := exec.Command(m.config.Beegfs, "--clientstats",
+			"--nodetype=meta", mountoption, "--allstats")
+		cmd.Stdin = strings.NewReader("\n")
+		cmdStdout := new(bytes.Buffer)
+		cmdStderr := new(bytes.Buffer)
+		cmd.Stdout = cmdStdout
+		cmd.Stderr = cmdStderr
+		err := cmd.Run()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error())
+			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode())
+			data, _ := ioutil.ReadAll(cmdStderr)
+			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stderr: \"%s\"\n", string(data))
+			data, _ = ioutil.ReadAll(cmdStdout)
+			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stdout: \"%s\"\n", string(data))
+			return
+		}
+		// Read I/O statistics
+		scanner := bufio.NewScanner(cmdStdout)
+
+		sumLine := regexp.MustCompile(`^Sum:\s+\d+\s+\[[a-zA-Z]+\]+`)
+		//Line := regexp.MustCompile(`^(.*)\s+(\d)+\s+\[([a-zA-Z]+)\]+`)
+		statsLine := regexp.MustCompile(`^(.*?)\s+?(\d.*?)$`)
+		singleSpacePattern := regexp.MustCompile(`\s+`)
+		removePattern := regexp.MustCompile(`[\[|\]]`)
+
+		for scanner.Scan() {
+			readLine := scanner.Text()
+			//fmt.Println(readLine)
+			// Jump few lines, we only want the I/O stats from nodes
+			if !sumLine.MatchString(readLine) {
+				continue
+			}
+
+			match := statsLine.FindStringSubmatch(readLine)
+			// nodeName = "Sum:" or would be nodes
+			// nodeName := match[1]
+			//Remove multiple whitespaces
+			dummy := removePattern.ReplaceAllString(match[2], " ")
+			metaStats := strings.TrimSpace(singleSpacePattern.ReplaceAllString(dummy, " "))
+			split := strings.Split(metaStats, " ")
+
+			// fill map with values
+			// split[i+1] = mdname
+			// split[i] = amount of md operations
+			for i := 0; i <= len(split)-1; i += 2 {
+				if _, ok := m.matches[split[i+1]]; ok {
+					m.matches["beegfs_cmeta_"+split[i+1]] = split[i]
+				} else {
+					f1, err := strconv.ParseFloat(m.matches["other"], 32)
+					if err != nil {
+						cclog.ComponentError(
+							m.name,
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
+						continue
+					}
+					f2, err := strconv.ParseFloat(split[i], 32)
+					if err != nil {
+						cclog.ComponentError(
+							m.name,
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
+						continue
+					}
+					//mdStat["other"] = fmt.Sprintf("%f", f1+f2)
+					m.matches["beegfs_cstorage_other"] = fmt.Sprintf("%f", f1+f2)
+				}
+			}
+
+			for key, data := range m.matches {
+				value, _ := strconv.ParseFloat(data, 32)
+				y, err := lp.New(key, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now())
+				if err == nil {
+					output <- y
+				}
+			}
+		}
+	}
+}
+
+func (m *BeegfsMetaCollector) Close() {
+	m.init = false
+}

collectors/beegfsmetaMetric.md

@@ -0,0 +1,75 @@
+## `BeeGFS on Demand` collector
+This Collector is to collect BeeGFS on Demand (BeeOND) metadata clientstats.
+
+```json
+  "beegfs_meta": {
+	"beegfs_path": "/usr/bin/beegfs-ctl",
+    "exclude_filesystem": [
+      "/mnt/ignore_me"
+    ],
+    "exclude_metrics": [     
+          "ack",
+          "entInf",
+          "fndOwn"
+    ]
+  }
+```
+
+The `BeeGFS On Demand (BeeOND)` collector uses the `beegfs-ctl` command to read performance metrics for
+BeeGFS filesystems.
+
+The reported filesystems can be filtered with the `exclude_filesystem` option
+in the configuration.
+
+The path to the `beegfs-ctl` command can be configured with the `beegfs_path` option
+in the configuration.
+
+When using the `exclude_metrics` option, the excluded metrics are summed as `other`.
+
+Important: The metrics listed below, are similar to the naming of BeeGFS. The Collector prefixes these with `beegfs_cstorage`(beegfs client storage).
+
+For example beegfs metric `open`-> `beegfs_cstorage_open`
+
+Available Metrics:
+
+* sum
+* ack
+* close
+* entInf
+* fndOwn
+* mkdir
+* create
+* rddir
+* refrEnt
+* mdsInf
+* rmdir
+* rmLnk
+* mvDirIns
+* mvFiIns
+* open
+* ren
+* sChDrct
+* sAttr
+* sDirPat
+* stat
+* statfs
+* trunc
+* symlnk
+* unlnk
+* lookLI
+* statLI
+* revalLI
+* openLI
+* createLI
+* hardlnk
+* flckAp
+* flckEn
+* flckRg
+* dirparent
+* listXA
+* getXA
+* rmXA
+* setXA
+* mirror
+
+The collector adds a `filesystem` tag to all metrics

collectors/beegfsstorageMetric.go

@@ -0,0 +1,221 @@
+package collectors
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"os/user"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+)
+
+// Struct for the collector-specific JSON config
+type BeegfsStorageCollectorConfig struct {
+	Beegfs            string   `json:"beegfs_path"`
+	ExcludeMetrics    []string `json:"exclude_metrics,omitempty"`
+	ExcludeFilesystem []string `json:"exclude_filesystem"`
+}
+
+type BeegfsStorageCollector struct {
+	metricCollector
+	tags    map[string]string
+	matches map[string]string
+	config  BeegfsStorageCollectorConfig
+	skipFS  map[string]struct{}
+}
+
+func (m *BeegfsStorageCollector) Init(config json.RawMessage) error {
+	// Check if already initialized
+	if m.init {
+		return nil
+	}
+	// Metrics
+	var storageStat_array = [18]string{
+		"sum", "ack", "sChDrct", "getFSize",
+		"sAttr", "statfs", "trunc", "close",
+		"fsync", "ops-rd", "MiB-rd/s", "ops-wr",
+		"MiB-wr/s", "gendbg", "hrtbeat", "remNode",
+		"storInf", "unlnk"}
+
+	m.name = "BeegfsStorageCollector"
+	m.setup()
+	// Set default beegfs-ctl binary
+
+	m.config.Beegfs = DEFAULT_BEEGFS_CMD
+
+	// Read JSON configuration
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &m.config)
+		if err != nil {
+			return err
+		}
+	}
+	println(m.config.Beegfs)
+	//create map with possible variables
+	m.matches = make(map[string]string)
+	for _, value := range storageStat_array {
+		_, skip := stringArrayContains(m.config.ExcludeMetrics, value)
+		if skip {
+			m.matches["other"] = "0"
+		} else {
+			m.matches["beegfs_cstorage_"+value] = "0"
+		}
+	}
+
+	m.meta = map[string]string{
+		"source": m.name,
+		"group":  "BeegfsStorage",
+	}
+	m.tags = map[string]string{
+		"type":       "node",
+		"filesystem": "",
+	}
+	m.skipFS = make(map[string]struct{})
+	for _, fs := range m.config.ExcludeFilesystem {
+		m.skipFS[fs] = struct{}{}
+	}
+
+	// Beegfs file system statistics can only be queried by user root
+	user, err := user.Current()
+	if err != nil {
+		return fmt.Errorf("BeegfsStorageCollector.Init(): Failed to get current user: %v", err)
+	}
+	if user.Uid != "0" {
+		return fmt.Errorf("BeegfsStorageCollector.Init(): BeeGFS file system statistics can only be queried by user root")
+	}
+
+	// Check if beegfs-ctl is in executable search path
+	_, err = exec.LookPath(m.config.Beegfs)
+	if err != nil {
+		return fmt.Errorf("BeegfsStorageCollector.Init(): Failed to find beegfs-ctl binary '%s': %v", m.config.Beegfs, err)
+	}
+	m.init = true
+	return nil
+}
+
+func (m *BeegfsStorageCollector) Read(interval time.Duration, output chan lp.CCMetric) {
+	if !m.init {
+		return
+	}
+	//get mounpoint
+	buffer, _ := ioutil.ReadFile(string("/proc/mounts"))
+	mounts := strings.Split(string(buffer), "\n")
+	var mountpoints []string
+	for _, line := range mounts {
+		if len(line) == 0 {
+			continue
+		}
+		f := strings.Fields(line)
+		if strings.Contains(f[0], "beegfs_ondemand") {
+			// Skip excluded filesystems
+			if _, skip := m.skipFS[f[1]]; skip {
+				continue
+			}
+			mountpoints = append(mountpoints, f[1])
+		}
+	}
+	if len(mountpoints) == 0 {
+		return
+	}
+	// collects stats for each BeeGFS on Demand FS
+	for _, mountpoint := range mountpoints {
+		m.tags["filesystem"] = mountpoint
+
+		// bwwgfs-ctl:
+		// --clientstats: Show client IO statistics.
+		// --nodetype=meta: The node type to query (meta, storage).
+		// --interval:
+		// --mount=/mnt/beeond/: Which mount point
+		//cmd := exec.Command(m.config.Beegfs, "/root/mc/test.txt")
+		mountoption := "--mount=" + mountpoint
+		cmd := exec.Command(m.config.Beegfs, "--clientstats",
+			"--nodetype=storage", mountoption, "--allstats")
+		cmd.Stdin = strings.NewReader("\n")
+		cmdStdout := new(bytes.Buffer)
+		cmdStderr := new(bytes.Buffer)
+		cmd.Stdout = cmdStdout
+		cmd.Stderr = cmdStderr
+		err := cmd.Run()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error())
+			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode())
+			data, _ := ioutil.ReadAll(cmdStderr)
+			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stderr: \"%s\"\n", string(data))
+			data, _ = ioutil.ReadAll(cmdStdout)
+			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stdout: \"%s\"\n", string(data))
+			return
+		}
+		// Read I/O statistics
+		scanner := bufio.NewScanner(cmdStdout)
+
+		sumLine := regexp.MustCompile(`^Sum:\s+\d+\s+\[[a-zA-Z]+\]+`)
+		//Line := regexp.MustCompile(`^(.*)\s+(\d)+\s+\[([a-zA-Z]+)\]+`)
+		statsLine := regexp.MustCompile(`^(.*?)\s+?(\d.*?)$`)
+		singleSpacePattern := regexp.MustCompile(`\s+`)
+		removePattern := regexp.MustCompile(`[\[|\]]`)
+
+		for scanner.Scan() {
+			readLine := scanner.Text()
+			//fmt.Println(readLine)
+			// Jump few lines, we only want the I/O stats from nodes
+			if !sumLine.MatchString(readLine) {
+				continue
+			}
+
+			match := statsLine.FindStringSubmatch(readLine)
+			// nodeName = "Sum:" or would be nodes
+			// nodeName := match[1]
+			//Remove multiple whitespaces
+			dummy := removePattern.ReplaceAllString(match[2], " ")
+			metaStats := strings.TrimSpace(singleSpacePattern.ReplaceAllString(dummy, " "))
+			split := strings.Split(metaStats, " ")
+
+			// fill map with values
+			// split[i+1] = mdname
+			// split[i] = amount of operations
+			for i := 0; i <= len(split)-1; i += 2 {
+				if _, ok := m.matches[split[i+1]]; ok {
+					m.matches["beegfs_cstorage_"+split[i+1]] = split[i]
+					//m.matches[split[i+1]] = split[i]
+				} else {
+					f1, err := strconv.ParseFloat(m.matches["other"], 32)
+					if err != nil {
+						cclog.ComponentError(
+							m.name,
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
+						continue
+					}
+					f2, err := strconv.ParseFloat(split[i], 32)
+					if err != nil {
+						cclog.ComponentError(
+							m.name,
+							fmt.Sprintf("Metric (other): Failed to convert str written '%s' to float: %v", m.matches["other"], err))
+						continue
+					}
+					m.matches["beegfs_cstorage_other"] = fmt.Sprintf("%f", f1+f2)
+				}
+			}
+
+			for key, data := range m.matches {
+				value, _ := strconv.ParseFloat(data, 32)
+				y, err := lp.New(key, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now())
+				if err == nil {
+					output <- y
+				}
+			}
+		}
+	}
+}
+
+func (m *BeegfsStorageCollector) Close() {
+	m.init = false
+}

collectors/beegfsstorageMetric.md

@@ -0,0 +1,55 @@
+## `BeeGFS on Demand` collector
+This Collector is to collect BeeGFS on Demand (BeeOND) storage stats.
+
+```json
+  "beegfs_storage": {
+	"beegfs_path": "/usr/bin/beegfs-ctl",
+    "exclude_filesystem": [
+      "/mnt/ignore_me"
+    ],
+    "exclude_metrics": [     
+          "ack",
+		  "storInf",
+		  "unlnk"
+    ]
+  }
+```
+
+The `BeeGFS On Demand (BeeOND)` collector uses the `beegfs-ctl` command to read performance metrics for BeeGFS filesystems.
+
+The reported filesystems can be filtered with the `exclude_filesystem` option
+in the configuration.
+
+The path to the `beegfs-ctl` command can be configured with the `beegfs_path` option
+in the configuration.
+
+When using the `exclude_metrics` option, the excluded metrics are summed as `other`.
+
+Important: The metrics listed below, are similar to the naming of BeeGFS. The Collector prefixes these with `beegfs_cstorage_`(beegfs client meta).
+For example beegfs metric `open`-> `beegfs_cstorage_`
+
+Note: BeeGFS FS offers many Metadata Information. Probably it makes sense to exlcude most of them. Nevertheless, these excluded metrics will be summed as `beegfs_cstorage_other`. 
+
+Available Metrics:
+
+* "sum"
+* "ack"
+* "sChDrct" 
+* "getFSize"
+* "sAttr"
+* "statfs"
+* "trunc"
+* "close"
+* "fsync"
+* "ops-rd"
+* "MiB-rd/s" 
+* "ops-wr"
+* "MiB-wr/s" 
+* "endbg" 
+* "hrtbeat"
+* "remNode"
+* "storInf"
+* "unlnk"
+
+
+The collector adds a `filesystem` tag to all metrics

collectors/collectorManager.go

@@ -19,7 +19,6 @@ var AvailableCollectors = map[string]MetricCollector{
 	"memstat":          new(MemstatCollector),
 	"netstat":          new(NetstatCollector),
 	"ibstat":           new(InfinibandCollector),
-	"ibstat_perfquery": new(InfinibandPerfQueryCollector),
 	"lustrestat":       new(LustreCollector),
 	"cpustat":          new(CpustatCollector),
 	"topprocs":         new(TopProcsCollector),
@@ -35,6 +34,8 @@ var AvailableCollectors = map[string]MetricCollector{
 	"nfs3stat":         new(Nfs3Collector),
 	"nfs4stat":         new(Nfs4Collector),
 	"numastats":        new(NUMAStatsCollector),
+	"beegfs_meta":      new(BeegfsMetaCollector),
+	"beegfs_storage":   new(BeegfsStorageCollector),
 }
 
 // Metric collector manager data structure

collectors/cpufreqCpuinfoMetric.go

@@ -57,7 +57,7 @@ func (m *CPUFreqCpuInfoCollector) Init(config json.RawMessage) error {
 	const cpuInfoFile = "/proc/cpuinfo"
 	file, err := os.Open(cpuInfoFile)
 	if err != nil {
-		return fmt.Errorf("Failed to open file '%s': %v", cpuInfoFile, err)
+		return fmt.Errorf("failed to open file '%s': %v", cpuInfoFile, err)
 	}
 	defer file.Close()
 
@@ -106,14 +106,14 @@ func (m *CPUFreqCpuInfoCollector) Init(config json.RawMessage) error {
 			topology.coreID = coreID
 			topology.coreID_int, err = strconv.ParseInt(coreID, 10, 64)
 			if err != nil {
-				return fmt.Errorf("Unable to convert coreID '%s' to int64: %v", coreID, err)
+				return fmt.Errorf("unable to convert coreID '%s' to int64: %v", coreID, err)
 			}
 
 			// Physical package ID
 			topology.physicalPackageID = physicalPackageID
 			topology.physicalPackageID_int, err = strconv.ParseInt(physicalPackageID, 10, 64)
 			if err != nil {
-				return fmt.Errorf("Unable to convert physicalPackageID '%s' to int64: %v", physicalPackageID, err)
+				return fmt.Errorf("unable to convert physicalPackageID '%s' to int64: %v", physicalPackageID, err)
 			}
 
 			// increase maximun socket / package ID, when required

collectors/cpufreqMetric.go

@@ -70,10 +70,10 @@ func (m *CPUFreqCollector) Init(config json.RawMessage) error {
 	globPattern := filepath.Join(baseDir, "cpu[0-9]*")
 	cpuDirs, err := filepath.Glob(globPattern)
 	if err != nil {
-		return fmt.Errorf("Unable to glob files with pattern '%s': %v", globPattern, err)
+		return fmt.Errorf("unable to glob files with pattern '%s': %v", globPattern, err)
 	}
 	if cpuDirs == nil {
-		return fmt.Errorf("Unable to find any files with pattern '%s'", globPattern)
+		return fmt.Errorf("unable to find any files with pattern '%s'", globPattern)
 	}
 
 	// Initialize CPU topology
@@ -82,38 +82,38 @@ func (m *CPUFreqCollector) Init(config json.RawMessage) error {
 		processor := strings.TrimPrefix(cpuDir, "/sys/devices/system/cpu/cpu")
 		processor_int, err := strconv.ParseInt(processor, 10, 64)
 		if err != nil {
-			return fmt.Errorf("Unable to convert cpuID '%s' to int64: %v", processor, err)
+			return fmt.Errorf("unable to convert cpuID '%s' to int64: %v", processor, err)
 		}
 
 		// Read package ID
 		physicalPackageIDFile := filepath.Join(cpuDir, "topology", "physical_package_id")
 		line, err := ioutil.ReadFile(physicalPackageIDFile)
 		if err != nil {
-			return fmt.Errorf("Unable to read physical package ID from file '%s': %v", physicalPackageIDFile, err)
+			return fmt.Errorf("unable to read physical package ID from file '%s': %v", physicalPackageIDFile, err)
 		}
 		physicalPackageID := strings.TrimSpace(string(line))
 		physicalPackageID_int, err := strconv.ParseInt(physicalPackageID, 10, 64)
 		if err != nil {
-			return fmt.Errorf("Unable to convert packageID '%s' to int64: %v", physicalPackageID, err)
+			return fmt.Errorf("unable to convert packageID '%s' to int64: %v", physicalPackageID, err)
 		}
 
 		// Read core ID
 		coreIDFile := filepath.Join(cpuDir, "topology", "core_id")
 		line, err = ioutil.ReadFile(coreIDFile)
 		if err != nil {
-			return fmt.Errorf("Unable to read core ID from file '%s': %v", coreIDFile, err)
+			return fmt.Errorf("unable to read core ID from file '%s': %v", coreIDFile, err)
 		}
 		coreID := strings.TrimSpace(string(line))
 		coreID_int, err := strconv.ParseInt(coreID, 10, 64)
 		if err != nil {
-			return fmt.Errorf("Unable to convert coreID '%s' to int64: %v", coreID, err)
+			return fmt.Errorf("unable to convert coreID '%s' to int64: %v", coreID, err)
 		}
 
 		// Check access to current frequency file
 		scalingCurFreqFile := filepath.Join(cpuDir, "cpufreq", "scaling_cur_freq")
 		err = unix.Access(scalingCurFreqFile, unix.R_OK)
 		if err != nil {
-			return fmt.Errorf("Unable to access file '%s': %v", scalingCurFreqFile, err)
+			return fmt.Errorf("unable to access file '%s': %v", scalingCurFreqFile, err)
 		}
 
 		t := &m.topology[processor_int]

collectors/cpustatMetric.go

@@ -21,11 +21,10 @@ type CpustatCollectorConfig struct {
 
 type CpustatCollector struct {
 	metricCollector
-	config          CpustatCollectorConfig
-	matches         map[string]int
-	cputags         map[string]map[string]string
-	nodetags        map[string]string
-	num_cpus_metric lp.CCMetric
+	config   CpustatCollectorConfig
+	matches  map[string]int
+	cputags  map[string]map[string]string
+	nodetags map[string]string
 }
 
 func (m *CpustatCollector) Init(config json.RawMessage) error {

collectors/customCmdMetric.go

@@ -61,7 +61,7 @@ func (m *CustomCmdCollector) Init(config json.RawMessage) error {
 		}
 	}
 	if len(m.files) == 0 && len(m.commands) == 0 {
-		return errors.New("No metrics to collect")
+		return errors.New("no metrics to collect")
 	}
 	m.handler = influx.NewMetricHandler()
 	m.parser = influx.NewParser(m.handler)

collectors/diskstatMetric.go

@@ -102,7 +102,7 @@ func (m *DiskstatCollector) Read(interval time.Duration, output chan lp.CCMetric
 			part_max_used = perc
 		}
 	}
-	y, err := lp.New("part_max_used", map[string]string{"type": "node"}, m.meta, map[string]interface{}{"value": part_max_used}, time.Now())
+	y, err := lp.New("part_max_used", map[string]string{"type": "node"}, m.meta, map[string]interface{}{"value": int(part_max_used)}, time.Now())
 	if err == nil {
 		y.AddMeta("unit", "percent")
 		output <- y

collectors/gpfsMetric.go

@@ -17,14 +17,24 @@ import (
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
 )
 
+const DEFAULT_GPFS_CMD = "mmpmon"
+
+type GpfsCollectorLastState struct {
+	bytesRead    int64
+	bytesWritten int64
+}
+
 type GpfsCollector struct {
 	metricCollector
 	tags   map[string]string
 	config struct {
 		Mmpmon            string   `json:"mmpmon_path,omitempty"`
 		ExcludeFilesystem []string `json:"exclude_filesystem,omitempty"`
+		SendBandwidths    bool     `json:"send_bandwidths"`
 	}
-	skipFS map[string]struct{}
+	skipFS        map[string]struct{}
+	lastTimestamp time.Time // Store time stamp of last tick to derive bandwidths
+	lastState     map[string]GpfsCollectorLastState
 }
 
 func (m *GpfsCollector) Init(config json.RawMessage) error {
@@ -38,7 +48,7 @@ func (m *GpfsCollector) Init(config json.RawMessage) error {
 	m.setup()
 
 	// Set default mmpmon binary
-	m.config.Mmpmon = "/usr/lpp/mmfs/bin/mmpmon"
+	m.config.Mmpmon = DEFAULT_GPFS_CMD
 
 	// Read JSON configuration
 	if len(config) > 0 {
@@ -64,17 +74,18 @@ func (m *GpfsCollector) Init(config json.RawMessage) error {
 	// GPFS / IBM Spectrum Scale file system statistics can only be queried by user root
 	user, err := user.Current()
 	if err != nil {
-		return fmt.Errorf("Failed to get current user: %v", err)
+		return fmt.Errorf("failed to get current user: %v", err)
 	}
 	if user.Uid != "0" {
 		return fmt.Errorf("GPFS file system statistics can only be queried by user root")
 	}
 
 	// Check if mmpmon is in executable search path
-	_, err = exec.LookPath(m.config.Mmpmon)
+	p, err := exec.LookPath(m.config.Mmpmon)
 	if err != nil {
-		return fmt.Errorf("Failed to find mmpmon binary '%s': %v", m.config.Mmpmon, err)
+		return fmt.Errorf("failed to find mmpmon binary '%s': %v", m.config.Mmpmon, err)
 	}
+	m.config.Mmpmon = p
 
 	m.init = true
 	return nil
@@ -86,6 +97,13 @@ func (m *GpfsCollector) Read(interval time.Duration, output chan lp.CCMetric) {
 		return
 	}
 
+	// Current time stamp
+	now := time.Now()
+	// time difference to last time stamp
+	timeDiff := now.Sub(m.lastTimestamp).Seconds()
+	// Save current timestamp
+	m.lastTimestamp = now
+
 	// mmpmon:
 	// -p: generate output that can be parsed
 	// -s: suppress the prompt on input
@@ -145,6 +163,12 @@ func (m *GpfsCollector) Read(interval time.Duration, output chan lp.CCMetric) {
 		}
 
 		m.tags["filesystem"] = filesystem
+		if _, ok := m.lastState[filesystem]; !ok {
+			m.lastState[filesystem] = GpfsCollectorLastState{
+				bytesRead:    -1,
+				bytesWritten: -1,
+			}
+		}
 
 		// return code
 		rc, err := strconv.Atoi(key_value["_rc_"])
@@ -188,6 +212,14 @@ func (m *GpfsCollector) Read(interval time.Duration, output chan lp.CCMetric) {
 		if y, err := lp.New("gpfs_bytes_read", m.tags, m.meta, map[string]interface{}{"value": bytesRead}, timestamp); err == nil {
 			output <- y
 		}
+		if m.config.SendBandwidths {
+			if lastBytesRead := m.lastState[filesystem].bytesRead; lastBytesRead >= 0 {
+				bwRead := float64(bytesRead-lastBytesRead) / timeDiff
+				if y, err := lp.New("gpfs_bw_read", m.tags, m.meta, map[string]interface{}{"value": bwRead}, timestamp); err == nil {
+					output <- y
+				}
+			}
+		}
 
 		// bytes written
 		bytesWritten, err := strconv.ParseInt(key_value["_bw_"], 10, 64)
@@ -200,6 +232,21 @@ func (m *GpfsCollector) Read(interval time.Duration, output chan lp.CCMetric) {
 		if y, err := lp.New("gpfs_bytes_written", m.tags, m.meta, map[string]interface{}{"value": bytesWritten}, timestamp); err == nil {
 			output <- y
 		}
+		if m.config.SendBandwidths {
+			if lastBytesWritten := m.lastState[filesystem].bytesRead; lastBytesWritten >= 0 {
+				bwWrite := float64(bytesWritten-lastBytesWritten) / timeDiff
+				if y, err := lp.New("gpfs_bw_write", m.tags, m.meta, map[string]interface{}{"value": bwWrite}, timestamp); err == nil {
+					output <- y
+				}
+			}
+		}
+
+		if m.config.SendBandwidths {
+			m.lastState[filesystem] = GpfsCollectorLastState{
+				bytesRead:    bytesRead,
+				bytesWritten: bytesWritten,
+			}
+		}
 
 		// number of opens
 		numOpens, err := strconv.ParseInt(key_value["_oc_"], 10, 64)

collectors/gpfsMetric.md

@@ -5,7 +5,8 @@
     "mmpmon_path": "/path/to/mmpmon",
     "exclude_filesystem": [
       "fs1"
-    ]
+    ],
+    "send_bandwidths" : true
   }
 ```
 
@@ -16,15 +17,18 @@ The reported filesystems can be filtered with the `exclude_filesystem` option
 in the configuration.
 
 The path to the `mmpmon` command can be configured with the `mmpmon_path` option
-in the configuration.
+in the configuration. If nothing is set, the collector searches in `$PATH` for `mmpmon`.
+
 
 Metrics:
-* `bytes_read`
+* `gpfs_bytes_read`
 * `gpfs_bytes_written`
 * `gpfs_num_opens`
 * `gpfs_num_closes`
 * `gpfs_num_reads`
 * `gpfs_num_readdirs`
 * `gpfs_num_inode_updates`
+* `gpfs_bw_read` (if `send_bandwidths == true`)
+* `gpfs_bw_write` (if `send_bandwidths == true`)
 
 The collector adds a `filesystem` tag to all metrics

collectors/infinibandMetric.go

@@ -16,7 +16,7 @@ import (
 	"time"
 )
 
-const IB_BASEPATH = `/sys/class/infiniband/`
+const IB_BASEPATH = "/sys/class/infiniband/"
 
 type InfinibandCollectorInfo struct {
 	LID              string            // IB local Identifier (LID)
@@ -24,14 +24,18 @@ type InfinibandCollectorInfo struct {
 	port             string            // IB device port
 	portCounterFiles map[string]string // mapping counter name -> sysfs file
 	tagSet           map[string]string // corresponding tag list
+	lastState        map[string]int64  // State from last measurement
 }
 
 type InfinibandCollector struct {
 	metricCollector
 	config struct {
-		ExcludeDevices []string `json:"exclude_devices,omitempty"` // IB device to exclude e.g. mlx5_0
+		ExcludeDevices     []string `json:"exclude_devices,omitempty"` // IB device to exclude e.g. mlx5_0
+		SendAbsoluteValues bool     `json:"send_abs_values"`           // Send absolut values as read from sys filesystem
+		SendDerivedValues  bool     `json:"send_derived_values"`       // Send derived values e.g. rates
 	}
-	info []*InfinibandCollectorInfo
+	info          []*InfinibandCollectorInfo
+	lastTimestamp time.Time // Store time stamp of last tick to derive bandwidths
 }
 
 // Init initializes the Infiniband collector by walking through files below IB_BASEPATH
@@ -49,6 +53,11 @@ func (m *InfinibandCollector) Init(config json.RawMessage) error {
 		"source": m.name,
 		"group":  "Network",
 	}
+
+	// Set default configuration,
+	m.config.SendAbsoluteValues = true
+	m.config.SendDerivedValues = false
+	// Read configuration file, allow overwriting default config
 	if len(config) > 0 {
 		err = json.Unmarshal(config, &m.config)
 		if err != nil {
@@ -60,10 +69,10 @@ func (m *InfinibandCollector) Init(config json.RawMessage) error {
 	globPattern := filepath.Join(IB_BASEPATH, "*", "ports", "*")
 	ibDirs, err := filepath.Glob(globPattern)
 	if err != nil {
-		return fmt.Errorf("Unable to glob files with pattern %s: %v", globPattern, err)
+		return fmt.Errorf("unable to glob files with pattern %s: %v", globPattern, err)
 	}
 	if ibDirs == nil {
-		return fmt.Errorf("Unable to find any directories with pattern %s", globPattern)
+		return fmt.Errorf("unable to find any directories with pattern %s", globPattern)
 	}
 
 	for _, path := range ibDirs {
@@ -106,10 +115,16 @@ func (m *InfinibandCollector) Init(config json.RawMessage) error {
 		for _, counterFile := range portCounterFiles {
 			err := unix.Access(counterFile, unix.R_OK)
 			if err != nil {
-				return fmt.Errorf("Unable to access %s: %v", counterFile, err)
+				return fmt.Errorf("unable to access %s: %v", counterFile, err)
 			}
 		}
 
+		// Initialize last state
+		lastState := make(map[string]int64)
+		for counter := range portCounterFiles {
+			lastState[counter] = -1
+		}
+
 		m.info = append(m.info,
 			&InfinibandCollectorInfo{
 				LID:              LID,
@@ -122,11 +137,12 @@ func (m *InfinibandCollector) Init(config json.RawMessage) error {
 					"port":   port,
 					"lid":    LID,
 				},
+				lastState: lastState,
 			})
 	}
 
 	if len(m.info) == 0 {
-		return fmt.Errorf("Found no IB devices")
+		return fmt.Errorf("found no IB devices")
 	}
 
 	m.init = true
@@ -141,9 +157,17 @@ func (m *InfinibandCollector) Read(interval time.Duration, output chan lp.CCMetr
 		return
 	}
 
+	// Current time stamp
 	now := time.Now()
+	// time difference to last time stamp
+	timeDiff := now.Sub(m.lastTimestamp).Seconds()
+	// Save current timestamp
+	m.lastTimestamp = now
+
 	for _, info := range m.info {
 		for counterName, counterFile := range info.portCounterFiles {
+
+			// Read counter file
 			line, err := ioutil.ReadFile(counterFile)
 			if err != nil {
 				cclog.ComponentError(
@@ -152,6 +176,8 @@ func (m *InfinibandCollector) Read(interval time.Duration, output chan lp.CCMetr
 				continue
 			}
 			data := strings.TrimSpace(string(line))
+
+			// convert counter to int64
 			v, err := strconv.ParseInt(data, 10, 64)
 			if err != nil {
 				cclog.ComponentError(
@@ -159,8 +185,24 @@ func (m *InfinibandCollector) Read(interval time.Duration, output chan lp.CCMetr
 					fmt.Sprintf("Read(): Failed to convert Infininiband metrice %s='%s' to int64: %v", counterName, data, err))
 				continue
 			}
-			if y, err := lp.New(counterName, info.tagSet, m.meta, map[string]interface{}{"value": v}, now); err == nil {
-				output <- y
+
+			// Send absolut values
+			if m.config.SendAbsoluteValues {
+				if y, err := lp.New(counterName, info.tagSet, m.meta, map[string]interface{}{"value": v}, now); err == nil {
+					output <- y
+				}
+			}
+
+			// Send derived values
+			if m.config.SendDerivedValues {
+				if info.lastState[counterName] >= 0 {
+					rate := float64((v - info.lastState[counterName])) / timeDiff
+					if y, err := lp.New(counterName+"_bw", info.tagSet, m.meta, map[string]interface{}{"value": rate}, now); err == nil {
+						output <- y
+					}
+				}
+				// Save current state
+				info.lastState[counterName] = v
 			}
 		}
 

collectors/infinibandMetric.md

@@ -5,7 +5,9 @@
   "ibstat": {
     "exclude_devices": [
       "mlx4"
-    ]
+    ],
+    "send_abs_values": true,
+    "send_derived_values": true
   }
 ```
 
@@ -22,5 +24,9 @@ Metrics:
 * `ib_xmit`
 * `ib_recv_pkts`
 * `ib_xmit_pkts`
+* `ib_recv_bw` (if `send_derived_values == true`)
+* `ib_xmit_bw` (if `send_derived_values == true`)
+* `ib_recv_pkts_bw` (if `send_derived_values == true`)
+* `ib_xmit_pkts_bw` (if `send_derived_values == true`)
 
 The collector adds a `device` tag to all metrics

collectors/infinibandPerfQueryMetric.go

@@ -1,232 +0,0 @@
-package collectors
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os/exec"
-
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
-
-	//	"os"
-	"encoding/json"
-	"errors"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"time"
-)
-
-const PERFQUERY = `/usr/sbin/perfquery`
-
-type InfinibandPerfQueryCollector struct {
-	metricCollector
-	tags   map[string]string
-	lids   map[string]map[string]string
-	config struct {
-		ExcludeDevices []string `json:"exclude_devices,omitempty"`
-		PerfQueryPath  string   `json:"perfquery_path"`
-	}
-}
-
-func (m *InfinibandPerfQueryCollector) Init(config json.RawMessage) error {
-	var err error
-	m.name = "InfinibandCollectorPerfQuery"
-	m.setup()
-	m.meta = map[string]string{"source": m.name, "group": "Network"}
-	m.tags = map[string]string{"type": "node"}
-	if len(config) > 0 {
-		err = json.Unmarshal(config, &m.config)
-		if err != nil {
-			return err
-		}
-	}
-	if len(m.config.PerfQueryPath) == 0 {
-		path, err := exec.LookPath("perfquery")
-		if err == nil {
-			m.config.PerfQueryPath = path
-		}
-	}
-	m.lids = make(map[string]map[string]string)
-	p := fmt.Sprintf("%s/*/ports/*/lid", string(IB_BASEPATH))
-	files, err := filepath.Glob(p)
-	if err != nil {
-		return err
-	}
-	for _, f := range files {
-		lid, err := ioutil.ReadFile(f)
-		if err == nil {
-			plist := strings.Split(strings.Replace(f, string(IB_BASEPATH), "", -1), "/")
-			skip := false
-			for _, d := range m.config.ExcludeDevices {
-				if d == plist[0] {
-					skip = true
-				}
-			}
-			if !skip {
-				m.lids[plist[0]] = make(map[string]string)
-				m.lids[plist[0]][plist[2]] = string(lid)
-			}
-		}
-	}
-
-	for _, ports := range m.lids {
-		for port, lid := range ports {
-			args := fmt.Sprintf("-r %s %s 0xf000", lid, port)
-			command := exec.Command(m.config.PerfQueryPath, args)
-			command.Wait()
-			_, err := command.Output()
-			if err != nil {
-				return fmt.Errorf("Failed to execute %s: %v", m.config.PerfQueryPath, err)
-			}
-		}
-	}
-
-	if len(m.lids) == 0 {
-		return errors.New("No usable IB devices")
-	}
-
-	m.init = true
-	return nil
-}
-
-func (m *InfinibandPerfQueryCollector) doPerfQuery(cmd string, dev string, lid string, port string, tags map[string]string, output chan lp.CCMetric) error {
-
-	args := fmt.Sprintf("-r %s %s 0xf000", lid, port)
-	command := exec.Command(cmd, args)
-	command.Wait()
-	stdout, err := command.Output()
-	if err != nil {
-		log.Print(err)
-		return err
-	}
-	ll := strings.Split(string(stdout), "\n")
-
-	for _, line := range ll {
-		if strings.HasPrefix(line, "PortRcvData") || strings.HasPrefix(line, "RcvData") {
-			lv := strings.Fields(line)
-			v, err := strconv.ParseFloat(lv[1], 64)
-			if err == nil {
-				y, err := lp.New("ib_recv", tags, m.meta, map[string]interface{}{"value": float64(v)}, time.Now())
-				if err == nil {
-					output <- y
-				}
-			}
-		}
-		if strings.HasPrefix(line, "PortXmitData") || strings.HasPrefix(line, "XmtData") {
-			lv := strings.Fields(line)
-			v, err := strconv.ParseFloat(lv[1], 64)
-			if err == nil {
-				y, err := lp.New("ib_xmit", tags, m.meta, map[string]interface{}{"value": float64(v)}, time.Now())
-				if err == nil {
-					output <- y
-				}
-			}
-		}
-		if strings.HasPrefix(line, "PortRcvPkts") || strings.HasPrefix(line, "RcvPkts") {
-			lv := strings.Fields(line)
-			v, err := strconv.ParseFloat(lv[1], 64)
-			if err == nil {
-				y, err := lp.New("ib_recv_pkts", tags, m.meta, map[string]interface{}{"value": float64(v)}, time.Now())
-				if err == nil {
-					output <- y
-				}
-			}
-		}
-		if strings.HasPrefix(line, "PortXmitPkts") || strings.HasPrefix(line, "XmtPkts") {
-			lv := strings.Fields(line)
-			v, err := strconv.ParseFloat(lv[1], 64)
-			if err == nil {
-				y, err := lp.New("ib_xmit_pkts", tags, m.meta, map[string]interface{}{"value": float64(v)}, time.Now())
-				if err == nil {
-					output <- y
-				}
-			}
-		}
-		if strings.HasPrefix(line, "PortRcvPkts") || strings.HasPrefix(line, "RcvPkts") {
-			lv := strings.Fields(line)
-			v, err := strconv.ParseFloat(lv[1], 64)
-			if err == nil {
-				y, err := lp.New("ib_recv_pkts", tags, m.meta, map[string]interface{}{"value": float64(v)}, time.Now())
-				if err == nil {
-					output <- y
-				}
-			}
-		}
-		if strings.HasPrefix(line, "PortXmitPkts") || strings.HasPrefix(line, "XmtPkts") {
-			lv := strings.Fields(line)
-			v, err := strconv.ParseFloat(lv[1], 64)
-			if err == nil {
-				y, err := lp.New("ib_xmit_pkts", tags, m.meta, map[string]interface{}{"value": float64(v)}, time.Now())
-				if err == nil {
-					output <- y
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func (m *InfinibandPerfQueryCollector) Read(interval time.Duration, output chan lp.CCMetric) {
-
-	if m.init {
-		for dev, ports := range m.lids {
-			for port, lid := range ports {
-				tags := map[string]string{
-					"type":   "node",
-					"device": dev,
-					"port":   port,
-					"lid":    lid}
-				path := fmt.Sprintf("%s/%s/ports/%s/counters/", string(IB_BASEPATH), dev, port)
-				buffer, err := ioutil.ReadFile(fmt.Sprintf("%s/port_rcv_data", path))
-				if err == nil {
-					data := strings.Replace(string(buffer), "\n", "", -1)
-					v, err := strconv.ParseFloat(data, 64)
-					if err == nil {
-						y, err := lp.New("ib_recv", tags, m.meta, map[string]interface{}{"value": float64(v)}, time.Now())
-						if err == nil {
-							output <- y
-						}
-					}
-				}
-				buffer, err = ioutil.ReadFile(fmt.Sprintf("%s/port_xmit_data", path))
-				if err == nil {
-					data := strings.Replace(string(buffer), "\n", "", -1)
-					v, err := strconv.ParseFloat(data, 64)
-					if err == nil {
-						y, err := lp.New("ib_xmit", tags, m.meta, map[string]interface{}{"value": float64(v)}, time.Now())
-						if err == nil {
-							output <- y
-						}
-					}
-				}
-				buffer, err = ioutil.ReadFile(fmt.Sprintf("%s/port_rcv_packets", path))
-				if err == nil {
-					data := strings.Replace(string(buffer), "\n", "", -1)
-					v, err := strconv.ParseFloat(data, 64)
-					if err == nil {
-						y, err := lp.New("ib_recv_pkts", tags, m.meta, map[string]interface{}{"value": float64(v)}, time.Now())
-						if err == nil {
-							output <- y
-						}
-					}
-				}
-				buffer, err = ioutil.ReadFile(fmt.Sprintf("%s/port_xmit_packets", path))
-				if err == nil {
-					data := strings.Replace(string(buffer), "\n", "", -1)
-					v, err := strconv.ParseFloat(data, 64)
-					if err == nil {
-						y, err := lp.New("ib_xmit_pkts", tags, m.meta, map[string]interface{}{"value": float64(v)}, time.Now())
-						if err == nil {
-							output <- y
-						}
-					}
-				}
-			}
-		}
-	}
-}
-
-func (m *InfinibandPerfQueryCollector) Close() {
-	m.init = false
-}

collectors/infinibandPerfQueryMetric.md

@@ -1,28 +0,0 @@
-
-## `ibstat_perfquery` collector
-
-```json
-  "ibstat_perfquery": {
-    "perfquery_path": "/path/to/perfquery",
-    "exclude_devices": [
-      "mlx4"
-    ]
-  }
-```
-
-The `ibstat_perfquery` collector includes all Infiniband devices that can be
-found below `/sys/class/infiniband/` and where any of the ports provides a
-LID file (`/sys/class/infiniband/<dev>/ports/<port>/lid`)
-
-The devices can be filtered with the `exclude_devices` option in the configuration.
-
-For each found LID the collector calls the `perfquery` command. The path to the
-`perfquery` command can be configured with the `perfquery_path` option in the configuration
-
-Metrics:
-* `ib_recv`
-* `ib_xmit`
-* `ib_recv_pkts`
-* `ib_xmit_pkts`
-
-The collector adds a `device` tag to all metrics

collectors/ipmiMetric.go

@@ -54,7 +54,7 @@ func (m *IpmiCollector) Init(config json.RawMessage) error {
 		m.ipmisensors = p
 	}
 	if len(m.ipmitool) == 0 && len(m.ipmisensors) == 0 {
-		return errors.New("No IPMI reader found")
+		return errors.New("no IPMI reader found")
 	}
 	m.init = true
 	return nil

collectors/likwidMetric.go

@@ -2,7 +2,7 @@ package collectors
 
 /*
 #cgo CFLAGS: -I./likwid
-#cgo LDFLAGS: -L./likwid -llikwid -llikwid-hwloc -lm -Wl,--unresolved-symbols=ignore-in-object-files
+#cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files
 #include <stdlib.h>
 #include <likwid.h>
 */
@@ -15,7 +15,6 @@ import (
 	"io/ioutil"
 	"math"
 	"os"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -28,66 +27,23 @@ import (
 	"github.com/NVIDIA/go-nvml/pkg/dl"
 )
 
-type MetricScope string
-
 const (
-	METRIC_SCOPE_HWTHREAD = iota
-	METRIC_SCOPE_CORE
-	METRIC_SCOPE_LLC
-	METRIC_SCOPE_NUMA
-	METRIC_SCOPE_DIE
-	METRIC_SCOPE_SOCKET
-	METRIC_SCOPE_NODE
-)
-
-func (ms MetricScope) String() string {
-	return string(ms)
-}
-
-func (ms MetricScope) Likwid() string {
-	LikwidDomains := map[string]string{
-		"cpu":        "",
-		"core":       "",
-		"llc":        "C",
-		"numadomain": "M",
-		"die":        "D",
-		"socket":     "S",
-		"node":       "N",
-	}
-	return LikwidDomains[string(ms)]
-}
-
-func (ms MetricScope) Granularity() int {
-	for i, g := range GetAllMetricScopes() {
-		if ms == g {
-			return i
-		}
-	}
-	return -1
-}
-
-func GetAllMetricScopes() []MetricScope {
-	return []MetricScope{"cpu" /*, "core", "llc", "numadomain", "die",*/, "socket", "node"}
-}
-
-const (
-	LIKWID_LIB_NAME     = "liblikwid.so"
-	LIKWID_LIB_DL_FLAGS = dl.RTLD_LAZY | dl.RTLD_GLOBAL
+	LIKWID_LIB_NAME       = "liblikwid.so"
+	LIKWID_LIB_DL_FLAGS   = dl.RTLD_LAZY | dl.RTLD_GLOBAL
+	LIKWID_DEF_ACCESSMODE = "direct"
 )
 
 type LikwidCollectorMetricConfig struct {
-	Name string `json:"name"` // Name of the metric
-	Calc string `json:"calc"` // Calculation for the metric using
-	//Aggr        string      `json:"aggregation"` // if scope unequal to LIKWID metric scope, the values are combined (sum, min, max, mean or avg, median)
-	Scope       MetricScope `json:"scope"` // scope for calculation. subscopes are aggregated using the 'aggregation' function
-	Publish     bool        `json:"publish"`
-	granulatity MetricScope
+	Name    string `json:"name"` // Name of the metric
+	Calc    string `json:"calc"` // Calculation for the metric using
+	Type    string `json:"type"` // Metric type (aka node, socket, cpu, ...)
+	Publish bool   `json:"publish"`
+	Unit    string `json:"unit"` // Unit of metric if any
 }
 
 type LikwidCollectorEventsetConfig struct {
-	Events      map[string]string `json:"events"`
-	granulatity map[string]MetricScope
-	Metrics     []LikwidCollectorMetricConfig `json:"metrics"`
+	Events  map[string]string             `json:"events"`
+	Metrics []LikwidCollectorMetricConfig `json:"metrics"`
 }
 
 type LikwidCollectorConfig struct {
@@ -95,28 +51,30 @@ type LikwidCollectorConfig struct {
 	Metrics        []LikwidCollectorMetricConfig   `json:"globalmetrics,omitempty"`
 	ForceOverwrite bool                            `json:"force_overwrite,omitempty"`
 	InvalidToZero  bool                            `json:"invalid_to_zero,omitempty"`
+	AccessMode     string                          `json:"access_mode,omitempty"`
+	DaemonPath     string                          `json:"accessdaemon_path,omitempty"`
+	LibraryPath    string                          `json:"liblikwid_path,omitempty"`
 }
 
 type LikwidCollector struct {
 	metricCollector
-	cpulist       []C.int
-	cpu2tid       map[int]int
-	sock2tid      map[int]int
-	scopeRespTids map[MetricScope]map[int]int
-	metrics       map[C.int]map[string]int
-	groups        []C.int
-	config        LikwidCollectorConfig
-	results       map[int]map[int]map[string]interface{}
-	mresults      map[int]map[int]map[string]float64
-	gmresults     map[int]map[string]float64
-	basefreq      float64
-	running       bool
+	cpulist   []C.int
+	cpu2tid   map[int]int
+	sock2tid  map[int]int
+	metrics   map[C.int]map[string]int
+	groups    []C.int
+	config    LikwidCollectorConfig
+	results   map[int]map[int]map[string]interface{}
+	mresults  map[int]map[int]map[string]float64
+	gmresults map[int]map[string]float64
+	basefreq  float64
+	running   bool
 }
 
 type LikwidMetric struct {
 	name      string
 	search    string
-	scope     MetricScope
+	scope     string
 	group_idx int
 }
 
@@ -128,148 +86,45 @@ func eventsToEventStr(events map[string]string) string {
 	return strings.Join(elist, ",")
 }
 
-func getGranularity(counter, event string) MetricScope {
-	if strings.HasPrefix(counter, "PMC") || strings.HasPrefix(counter, "FIXC") {
-		return "cpu"
-	} else if strings.Contains(counter, "BOX") || strings.Contains(counter, "DEV") {
-		return "socket"
-	} else if strings.HasPrefix(counter, "PWR") {
-		if event == "RAPL_CORE_ENERGY" {
-			return "cpu"
-		} else {
-			return "socket"
-		}
-	}
-	return "unknown"
-}
-
 func getBaseFreq() float64 {
 	var freq float64 = math.NaN()
 	C.power_init(0)
 	info := C.get_powerInfo()
 	if float64(info.baseFrequency) != 0 {
-		freq = float64(info.baseFrequency) * 1e3
+		freq = float64(info.baseFrequency) * 1e6
 	} else {
 		buffer, err := ioutil.ReadFile("/sys/devices/system/cpu/cpu0/cpufreq/bios_limit")
 		if err == nil {
 			data := strings.Replace(string(buffer), "\n", "", -1)
 			x, err := strconv.ParseInt(data, 0, 64)
 			if err == nil {
-				freq = float64(x) * 1e3
+				freq = float64(x) * 1e6
 			}
 		}
 	}
 	return freq
 }
 
-func (m *LikwidCollector) initGranularity() {
-	splitRegex := regexp.MustCompile("[+-/*()]")
-	for _, evset := range m.config.Eventsets {
-		evset.granulatity = make(map[string]MetricScope)
-		for counter, event := range evset.Events {
-			gran := getGranularity(counter, event)
-			if gran.Granularity() >= 0 {
-				evset.granulatity[counter] = gran
-			}
-		}
-		for i, metric := range evset.Metrics {
-			s := splitRegex.Split(metric.Calc, -1)
-			gran := MetricScope("cpu")
-			evset.Metrics[i].granulatity = gran
-			for _, x := range s {
-				if _, ok := evset.Events[x]; ok {
-					if evset.granulatity[x].Granularity() > gran.Granularity() {
-						gran = evset.granulatity[x]
-					}
-				}
-			}
-			evset.Metrics[i].granulatity = gran
-		}
-	}
-	for i, metric := range m.config.Metrics {
-		s := splitRegex.Split(metric.Calc, -1)
-		gran := MetricScope("cpu")
-		m.config.Metrics[i].granulatity = gran
-		for _, x := range s {
-			for _, evset := range m.config.Eventsets {
-				for _, m := range evset.Metrics {
-					if m.Name == x && m.granulatity.Granularity() > gran.Granularity() {
-						gran = m.granulatity
-					}
-				}
-			}
-		}
-		m.config.Metrics[i].granulatity = gran
-	}
-}
-
-type TopoResolveFunc func(cpuid int) int
-
-func (m *LikwidCollector) getResponsiblities() map[MetricScope]map[int]int {
-	get_cpus := func(scope MetricScope) map[int]int {
-		var slist []int
-		var cpu C.int
-		var input func(index int) string
-		switch scope {
-		case "node":
-			slist = []int{0}
-			input = func(index int) string { return "N:0" }
-		case "socket":
-			input = func(index int) string { return fmt.Sprintf("%s%d:0", scope.Likwid(), index) }
-			slist = topo.SocketList()
-		// case "numadomain":
-		// 	input = func(index int) string { return fmt.Sprintf("%s%d:0", scope.Likwid(), index) }
-		// 	slist = topo.NumaNodeList()
-		// 	cclog.Debug(scope, " ", input(0), " ", slist)
-		// case "die":
-		// 	input = func(index int) string { return fmt.Sprintf("%s%d:0", scope.Likwid(), index) }
-		// 	slist = topo.DieList()
-		// case "llc":
-		// 	input = fmt.Sprintf("%s%d:0", scope.Likwid(), s)
-		// 	slist = topo.LLCacheList()
-		case "cpu":
-			input = func(index int) string { return fmt.Sprintf("%d", index) }
-			slist = topo.CpuList()
-		case "hwthread":
-			input = func(index int) string { return fmt.Sprintf("%d", index) }
-			slist = topo.CpuList()
-		}
-		outmap := make(map[int]int)
-		for _, s := range slist {
-			t := C.CString(input(s))
-			clen := C.cpustr_to_cpulist(t, &cpu, 1)
-			if int(clen) == 1 {
-				outmap[s] = m.cpu2tid[int(cpu)]
-			} else {
-				cclog.Error(fmt.Sprintf("Cannot determine responsible CPU for %s", input(s)))
-				outmap[s] = -1
-			}
-			C.free(unsafe.Pointer(t))
-		}
-		return outmap
-	}
-
-	scopes := GetAllMetricScopes()
-	complete := make(map[MetricScope]map[int]int)
-	for _, s := range scopes {
-		complete[s] = get_cpus(s)
-	}
-	return complete
-}
-
 func (m *LikwidCollector) Init(config json.RawMessage) error {
 	var ret C.int
 	m.name = "LikwidCollector"
+	m.config.AccessMode = LIKWID_DEF_ACCESSMODE
+	m.config.LibraryPath = LIKWID_LIB_NAME
 	if len(config) > 0 {
 		err := json.Unmarshal(config, &m.config)
 		if err != nil {
 			return err
 		}
 	}
-	lib := dl.New(LIKWID_LIB_NAME, LIKWID_LIB_DL_FLAGS)
+	lib := dl.New(m.config.LibraryPath, LIKWID_LIB_DL_FLAGS)
 	if lib == nil {
-		return fmt.Errorf("error instantiating DynamicLibrary for %s", LIKWID_LIB_NAME)
+		return fmt.Errorf("error instantiating DynamicLibrary for %s", m.config.LibraryPath)
 	}
+	err := lib.Open()
+	if err != nil {
+		return fmt.Errorf("error opening %s: %v", m.config.LibraryPath, err)
+	}
+
 	if m.config.ForceOverwrite {
 		cclog.ComponentDebug(m.name, "Set LIKWID_FORCE=1")
 		os.Setenv("LIKWID_FORCE", "1")
@@ -284,7 +139,16 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 	for i, c := range cpulist {
 		m.cpulist[i] = C.int(c)
 		m.cpu2tid[c] = i
-
+	}
+	m.sock2tid = make(map[int]int)
+	tmp := make([]C.int, 1)
+	for _, sid := range topo.SocketList() {
+		cstr := C.CString(fmt.Sprintf("S%d:0", sid))
+		ret = C.cpustr_to_cpulist(cstr, &tmp[0], 1)
+		if ret > 0 {
+			m.sock2tid[sid] = m.cpu2tid[int(tmp[0])]
+		}
+		C.free(unsafe.Pointer(cstr))
 	}
 	m.results = make(map[int]map[int]map[string]interface{})
 	m.mresults = make(map[int]map[int]map[string]float64)
@@ -297,10 +161,16 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 		return err
 	}
 
-	// Determine which counter works at which level. PMC*: cpu, *BOX*: socket, ...
-	m.initGranularity()
-	// Generate map for MetricScope -> scope_id (like socket id) -> responsible id (offset in cpulist)
-	m.scopeRespTids = m.getResponsiblities()
+	switch m.config.AccessMode {
+	case "direct":
+		C.HPMmode(0)
+	case "accessdaemon":
+		if len(m.config.DaemonPath) > 0 {
+			p := os.Getenv("PATH")
+			os.Setenv("PATH", m.config.DaemonPath+":"+p)
+		}
+		C.HPMmode(1)
+	}
 
 	cclog.ComponentDebug(m.name, "initialize LIKWID perfmon module")
 	ret = C.perfmon_init(C.int(len(m.cpulist)), &m.cpulist[0])
@@ -317,29 +187,36 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 	globalParams["inverseClock"] = float64(1.0)
 	// While adding the events, we test the metrics whether they can be computed at all
 	for i, evset := range m.config.Eventsets {
-		estr := eventsToEventStr(evset.Events)
-		// Generate parameter list for the metric computing test
-		params := make(map[string]interface{})
-		params["time"] = float64(1.0)
-		params["inverseClock"] = float64(1.0)
-		for counter := range evset.Events {
-			params[counter] = float64(1.0)
-		}
-		for _, metric := range evset.Metrics {
-			// Try to evaluate the metric
-			_, err := agg.EvalFloat64Condition(metric.Calc, params)
-			if err != nil {
-				cclog.ComponentError(m.name, "Calculation for metric", metric.Name, "failed:", err.Error())
-				continue
+		var gid C.int
+		var cstr *C.char
+		if len(evset.Events) > 0 {
+			estr := eventsToEventStr(evset.Events)
+			// Generate parameter list for the metric computing test
+			params := make(map[string]interface{})
+			params["time"] = float64(1.0)
+			params["inverseClock"] = float64(1.0)
+			for counter := range evset.Events {
+				params[counter] = float64(1.0)
 			}
-			// If the metric is not in the parameter list for the global metrics, add it
-			if _, ok := globalParams[metric.Name]; !ok {
-				globalParams[metric.Name] = float64(1.0)
+			for _, metric := range evset.Metrics {
+				// Try to evaluate the metric
+				_, err := agg.EvalFloat64Condition(metric.Calc, params)
+				if err != nil {
+					cclog.ComponentError(m.name, "Calculation for metric", metric.Name, "failed:", err.Error())
+					continue
+				}
+				// If the metric is not in the parameter list for the global metrics, add it
+				if _, ok := globalParams[metric.Name]; !ok {
+					globalParams[metric.Name] = float64(1.0)
+				}
 			}
+			// Now we add the list of events to likwid
+			cstr = C.CString(estr)
+			gid = C.perfmon_addEventSet(cstr)
+		} else {
+			cclog.ComponentError(m.name, "Invalid Likwid eventset config, no events given")
+			continue
 		}
-		// Now we add the list of events to likwid
-		cstr := C.CString(estr)
-		gid := C.perfmon_addEventSet(cstr)
 		if gid >= 0 {
 			m.groups = append(m.groups, gid)
 		}
@@ -415,15 +292,9 @@ func (m *LikwidCollector) calcEventsetMetrics(group int, interval time.Duration,
 	// Go over events and get the results
 	for eidx = 0; int(eidx) < len(evset.Events); eidx++ {
 		ctr := C.perfmon_getCounterName(gid, eidx)
-		ev := C.perfmon_getEventName(gid, eidx)
 		gctr := C.GoString(ctr)
-		gev := C.GoString(ev)
-		// MetricScope for the counter (and if needed the event)
-		scope := getGranularity(gctr, gev)
-		// Get the map scope-id -> tids
-		// This way we read less counters like only the responsible hardware thread for a socket
-		scopemap := m.scopeRespTids[scope]
-		for _, tid := range scopemap {
+
+		for _, tid := range m.cpu2tid {
 			if tid >= 0 {
 				m.results[group][tid]["time"] = interval.Seconds()
 				m.results[group][tid]["inverseClock"] = invClock
@@ -437,7 +308,10 @@ func (m *LikwidCollector) calcEventsetMetrics(group int, interval time.Duration,
 	for _, metric := range evset.Metrics {
 		// The metric scope is determined in the Init() function
 		// Get the map scope-id -> tids
-		scopemap := m.scopeRespTids[metric.Scope]
+		scopemap := m.cpu2tid
+		if metric.Type == "socket" {
+			scopemap = m.sock2tid
+		}
 		for domain, tid := range scopemap {
 			if tid >= 0 {
 				value, err := agg.EvalFloat64Condition(metric.Calc, m.results[group][tid])
@@ -455,13 +329,15 @@ func (m *LikwidCollector) calcEventsetMetrics(group int, interval time.Duration,
 				// Now we have the result, send it with the proper tags
 				if !math.IsNaN(value) {
 					if metric.Publish {
-						tags := map[string]string{"type": metric.Scope.String()}
-						if metric.Scope != "node" {
-							tags["type-id"] = fmt.Sprintf("%d", domain)
-						}
 						fields := map[string]interface{}{"value": value}
-						y, err := lp.New(metric.Name, tags, m.meta, fields, time.Now())
+						y, err := lp.New(metric.Name, map[string]string{"type": metric.Type}, m.meta, fields, time.Now())
 						if err == nil {
+							if metric.Type != "node" {
+								y.AddTag("type-id", fmt.Sprintf("%d", domain))
+							}
+							if len(metric.Unit) > 0 {
+								y.AddMeta("unit", metric.Unit)
+							}
 							output <- y
 						}
 					}
@@ -476,7 +352,10 @@ func (m *LikwidCollector) calcEventsetMetrics(group int, interval time.Duration,
 // Go over the global metrics, derive the value out of the event sets' metric values and send it
 func (m *LikwidCollector) calcGlobalMetrics(interval time.Duration, output chan lp.CCMetric) error {
 	for _, metric := range m.config.Metrics {
-		scopemap := m.scopeRespTids[metric.Scope]
+		scopemap := m.cpu2tid
+		if metric.Type == "socket" {
+			scopemap = m.sock2tid
+		}
 		for domain, tid := range scopemap {
 			if tid >= 0 {
 				// Here we generate parameter list
@@ -502,13 +381,16 @@ func (m *LikwidCollector) calcGlobalMetrics(interval time.Duration, output chan
 				// Now we have the result, send it with the proper tags
 				if !math.IsNaN(value) {
 					if metric.Publish {
-						tags := map[string]string{"type": metric.Scope.String()}
-						if metric.Scope != "node" {
-							tags["type-id"] = fmt.Sprintf("%d", domain)
-						}
+						tags := map[string]string{"type": metric.Type}
 						fields := map[string]interface{}{"value": value}
 						y, err := lp.New(metric.Name, tags, m.meta, fields, time.Now())
 						if err == nil {
+							if metric.Type != "node" {
+								y.AddTag("type-id", fmt.Sprintf("%d", domain))
+							}
+							if len(metric.Unit) > 0 {
+								y.AddMeta("unit", metric.Unit)
+							}
 							output <- y
 						}
 					}

collectors/likwidMetric.md

@@ -4,12 +4,17 @@
 The `likwid` collector is probably the most complicated collector. The LIKWID library is included as static library with *direct* access mode. The *direct* access mode is suitable if the daemon is executed by a root user. The static library does not contain the performance groups, so all information needs to be provided in the configuration.
 
 The `likwid` configuration consists of two parts, the "eventsets" and "globalmetrics":
-- An event set list itself has two parts, the "events" and a set of derivable "metrics". Each of the "events" is a counter:event pair in LIKWID's syntax. The "metrics" are a list of formulas to derive the metric value from the measurements of the "events". Each metric has a name, the formula, a scope and a publish flag. A counter names can be used like variables in the formulas, so `PMC0+PMC1` sums the measurements for the both events configured in the counters `PMC0` and `PMC1`. The scope tells the Collector whether it is a metric for each hardware thread (`cpu`) or each CPU socket (`socket`). The last one is the publishing flag. It tells the collector whether a metric should be sent to the router.
-- The global metrics are metrics which require data from all event set measurements to be derived. The inputs are the metrics in the event sets. Similar to the metrics in the event sets, the global metrics are defined by a name, a formula, a scope and a publish flag. See event set metrics for details. The only difference is that there is no access to the raw event measurements anymore but only to the metrics. So, the idea is to derive a metric in the "eventsets" section and reuse it in the "globalmetrics" part. If you need a metric only for deriving the global metrics, disable forwarding of the event set metrics. **Be aware** that the combination might be misleading because the "behavior" of a metric changes over time and the multiple measurements might count different computing phases.
+- An event set list itself has two parts, the "events" and a set of derivable "metrics". Each of the "events" is a counter:event pair in LIKWID's syntax. The "metrics" are a list of formulas to derive the metric value from the measurements of the "events". Each metric has a name, the formula, a scope and a publish flag. Counter names can be used like variables in the formulas, so `PMC0+PMC1` sums the measurements for the both events configured in the counters `PMC0` and `PMC1`. The scope tells the Collector whether it is a metric for each hardware thread (`cpu`) or each CPU socket (`socket`). You may specify a unit for the metric with `unit`. The last one is the publishing flag. It tells the collector whether a metric should be sent to the router.
+- The global metrics are metrics which require data from all event set measurements to be derived. The inputs are the metrics in the event sets. Similar to the metrics in the event sets, the global metrics are defined by a name, a formula, a scope and a publish flag. See event set metrics for details. The only difference is that there is no access to the raw event measurements anymore but only to the metrics. So, the idea is to derive a metric in the "eventsets" section and reuse it in the "globalmetrics" part. If you need a metric only for deriving the global metrics, disable forwarding of the event set metrics (`publish=false`). **Be aware** that the combination might be misleading because the "behavior" of a metric changes over time and the multiple measurements might count different computing phases. Similar to the metrics in the eventset, you can specify a metric unit with the `unit` field.
 
 Additional options:
+- `access_mode` : Method to use for hardware performance monitoring (`direct` access as root user, `accessdaemon` for the daemon mode)
+- `accessdaemon_path`: Folder with the access daemon `likwid-accessD`, commonly `$LIKWID_INSTALL_LOC/sbin`
 - `force_overwrite`: Same as setting `LIKWID_FORCE=1`. In case counters are already in-use, LIKWID overwrites their configuration to do its measurements
 - `invalid_to_zero`: In some cases, the calculations result in `NaN` or `Inf`. With this option, all `NaN` and `Inf` values are replaces with `0.0`.
+- `access_mode`: Specify LIKWID access mode: `direct` for direct register access as root user or `accessdaemon`
+- `accessdaemon_path`: Folder of the accessDaemon `likwid-accessD`
+- `liblikwid_path`: Location of `liblikwid.so`
 
 ### Available metric scopes
 
@@ -26,6 +31,57 @@ As a guideline:
 - All `PWRx` counters have scope `socket`, except `"PWR1" : "RAPL_CORE_ENERGY"` has `cpu` scope
 - All `DFCx` counters have scope `socket`
 
+### Help with the configuration
+
+The configuration for the `likwid` collector is quite complicated. Most users don't use LIKWID with the event:counter notation but rely on the performance groups defined by the LIKWID team for each architecture. In order to help with the `likwid` collector configuration, we included a script `scripts/likwid_perfgroup_to_cc_config.py` that creates the configuration of an `eventset` from a performance group (using a LIKWID installation in `$PATH`):
+```
+$ likwid-perfctr -i
+[...]
+short name: ICX
+[...]
+$ likwid-perfctr -a
+[...]
+MEM_DP
+MEM
+FLOPS_SP
+CLOCK
+[...]
+$ scripts/likwid_perfgroup_to_cc_config.py ICX MEM_DP
+{
+  "events": {
+    "FIXC0": "INSTR_RETIRED_ANY",
+    "..." : "..."
+  },
+  "metrics" : [
+    {
+      "calc": "time",
+      "name": "Runtime (RDTSC) [s]",
+      "publish": true,
+      "unit": "seconds"
+      "scope": "cpu"
+    },
+    {
+      "..." : "..."
+    }
+  ]
+}
+```
+
+You can copy this JSON and add it to the `eventsets` list. If you specify multiple event sets, you can add globally derived metrics in the extra `global_metrics` section with the metric names as variables.
+
+### Mixed usage between daemon and users
+
+LIKWID checks the file `/var/run/likwid.lock` before performing any interfering operations. Who is allowed to access the counters is determined by the owner of the file. If it does not exist, it is created for the current user. So, if you want to temporarly allow counter access to a user (e.g. in a job):
+
+Before (SLURM prolog, ...)
+```
+$ chwon $JOBUSER /var/run/likwid.lock
+```
+
+After (SLURM epilog, ...)
+```
+$ chwon $CCUSER /var/run/likwid.lock
+```
 
 ### Example configuration
 
@@ -52,25 +108,28 @@ As a guideline:
           {
             "name": "ipc",
             "calc": "PMC0/PMC1",
-            "scope": "cpu",
+            "type": "cpu",
             "publish": true
           },
           {
             "name": "flops_any",
             "calc": "0.000001*PMC2/time",
-            "scope": "cpu",
+            "unit": "MFlops/s",
+            "type": "cpu",
             "publish": true
           },
           {
-            "name": "clock_mhz",
+            "name": "clock",
             "calc": "0.000001*(FIXC1/FIXC2)/inverseClock",
-            "scope": "cpu",
+            "type": "cpu",
+            "unit": "MHz",
             "publish": true
           },
           {
             "name": "mem1",
             "calc": "0.000001*(DFC0+DFC1+DFC2+DFC3)*64.0/time",
-            "scope": "socket",
+            "unit": "Mbyte/s",
+            "type": "socket",
             "publish": false
           }
         ]
@@ -88,19 +147,22 @@ As a guideline:
           {
             "name": "pwr_core",
             "calc": "PWR0/time",
-            "scope": "socket",
+            "unit": "Watt"
+            "type": "socket",
             "publish": true
           },
           {
             "name": "pwr_pkg",
             "calc": "PWR1/time",
-            "scope": "socket",
+            "type": "socket",
+            "unit": "Watt"
             "publish": true
           },
           {
             "name": "mem2",
             "calc": "0.000001*(DFC0+DFC1+DFC2+DFC3)*64.0/time",
-            "scope": "socket",
+            "unit": "Mbyte/s",
+            "type": "socket",
             "publish": false
           }
         ]
@@ -110,7 +172,8 @@ As a guideline:
       {
         "name": "mem_bw",
         "calc": "mem1+mem2",
-        "scope": "socket",
+        "type": "socket",
+        "unit": "Mbyte/s",
         "publish": true
       }
     ]
@@ -146,3 +209,4 @@ IPC   PMC0/PMC1                  ->     {
                                  ->   ]
 ```
 
+The script `scripts/likwid_perfgroup_to_cc_config.py` might help you.

collectors/lustreMetric.go

@@ -19,23 +19,41 @@ const LCTL_CMD = `lctl`
 const LCTL_OPTION = `get_param`
 
 type LustreCollectorConfig struct {
-	LCtlCommand    string   `json:"lctl_command"`
-	ExcludeMetrics []string `json:"exclude_metrics"`
-	SendAllMetrics bool     `json:"send_all_metrics"`
+	LCtlCommand        string   `json:"lctl_command,omitempty"`
+	ExcludeMetrics     []string `json:"exclude_metrics,omitempty"`
+	Sudo               bool     `json:"use_sudo,omitempty"`
+	SendAbsoluteValues bool     `json:"send_abs_values,omitempty"`
+	SendDerivedValues  bool     `json:"send_derived_values,omitempty"`
+	SendDiffValues     bool     `json:"send_diff_values,omitempty"`
+}
+
+type LustreMetricDefinition struct {
+	name       string
+	lineprefix string
+	lineoffset int
+	unit       string
+	calc       string
 }
 
 type LustreCollector struct {
 	metricCollector
-	tags    map[string]string
-	matches map[string]map[string]int
-	stats   map[string]map[string]int64
-	config  LustreCollectorConfig
-	lctl    string
+	tags          map[string]string
+	config        LustreCollectorConfig
+	lctl          string
+	sudoCmd       string
+	lastTimestamp time.Time                   // Store time stamp of last tick to derive bandwidths
+	definitions   []LustreMetricDefinition    // Combined list without excluded metrics
+	stats         map[string]map[string]int64 // Data for last value per device and metric
 }
 
 func (m *LustreCollector) getDeviceDataCommand(device string) []string {
+	var command *exec.Cmd
 	statsfile := fmt.Sprintf("llite.%s.stats", device)
-	command := exec.Command(m.lctl, LCTL_OPTION, statsfile)
+	if m.config.Sudo {
+		command = exec.Command(m.sudoCmd, m.lctl, LCTL_OPTION, statsfile)
+	} else {
+		command = exec.Command(m.lctl, LCTL_OPTION, statsfile)
+	}
 	command.Wait()
 	stdout, _ := command.Output()
 	return strings.Split(string(stdout), "\n")
@@ -68,6 +86,16 @@ func (m *LustreCollector) getDevices() []string {
 	return devices
 }
 
+func getMetricData(lines []string, prefix string, offset int) (int64, error) {
+	for _, line := range lines {
+		if strings.HasPrefix(line, prefix) {
+			lf := strings.Fields(line)
+			return strconv.ParseInt(lf[offset], 0, 64)
+		}
+	}
+	return 0, errors.New("no such line in data")
+}
+
 // //Version reading the stats data of a device from sysfs
 // func (m *LustreCollector) getDeviceDataSysfs(device string) []string {
 // 	llitedir := filepath.Join(LUSTRE_SYSFS, "llite")
@@ -80,6 +108,183 @@ func (m *LustreCollector) getDevices() []string {
 // 	return strings.Split(string(buffer), "\n")
 // }
 
+var LustreAbsMetrics = []LustreMetricDefinition{
+	{
+		name:       "lustre_read_requests",
+		lineprefix: "read_bytes",
+		lineoffset: 1,
+		unit:       "requests",
+		calc:       "none",
+	},
+	{
+		name:       "lustre_write_requests",
+		lineprefix: "write_bytes",
+		lineoffset: 1,
+		unit:       "requests",
+		calc:       "none",
+	},
+	{
+		name:       "lustre_read_bytes",
+		lineprefix: "read_bytes",
+		lineoffset: 6,
+		unit:       "bytes",
+		calc:       "none",
+	},
+	{
+		name:       "lustre_write_bytes",
+		lineprefix: "write_bytes",
+		lineoffset: 6,
+		unit:       "bytes",
+		calc:       "none",
+	},
+	{
+		name:       "lustre_open",
+		lineprefix: "open",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "none",
+	},
+	{
+		name:       "lustre_close",
+		lineprefix: "close",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "none",
+	},
+	{
+		name:       "lustre_setattr",
+		lineprefix: "setattr",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "none",
+	},
+	{
+		name:       "lustre_getattr",
+		lineprefix: "getattr",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "none",
+	},
+	{
+		name:       "lustre_statfs",
+		lineprefix: "statfs",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "none",
+	},
+	{
+		name:       "lustre_inode_permission",
+		lineprefix: "inode_permission",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "none",
+	},
+}
+
+var LustreDiffMetrics = []LustreMetricDefinition{
+	{
+		name:       "lustre_read_requests_diff",
+		lineprefix: "read_bytes",
+		lineoffset: 1,
+		unit:       "requests",
+		calc:       "difference",
+	},
+	{
+		name:       "lustre_write_requests_diff",
+		lineprefix: "write_bytes",
+		lineoffset: 1,
+		unit:       "requests",
+		calc:       "difference",
+	},
+	{
+		name:       "lustre_read_bytes_diff",
+		lineprefix: "read_bytes",
+		lineoffset: 6,
+		unit:       "bytes",
+		calc:       "difference",
+	},
+	{
+		name:       "lustre_write_bytes_diff",
+		lineprefix: "write_bytes",
+		lineoffset: 6,
+		unit:       "bytes",
+		calc:       "difference",
+	},
+	{
+		name:       "lustre_open_diff",
+		lineprefix: "open",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "difference",
+	},
+	{
+		name:       "lustre_close_diff",
+		lineprefix: "close",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "difference",
+	},
+	{
+		name:       "lustre_setattr_diff",
+		lineprefix: "setattr",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "difference",
+	},
+	{
+		name:       "lustre_getattr_diff",
+		lineprefix: "getattr",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "difference",
+	},
+	{
+		name:       "lustre_statfs_diff",
+		lineprefix: "statfs",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "difference",
+	},
+	{
+		name:       "lustre_inode_permission_diff",
+		lineprefix: "inode_permission",
+		lineoffset: 1,
+		unit:       "",
+		calc:       "difference",
+	},
+}
+
+var LustreDeriveMetrics = []LustreMetricDefinition{
+	{
+		name:       "lustre_read_requests_rate",
+		lineprefix: "read_bytes",
+		lineoffset: 1,
+		unit:       "requests/sec",
+		calc:       "derivative",
+	},
+	{
+		name:       "lustre_write_requests_rate",
+		lineprefix: "write_bytes",
+		lineoffset: 1,
+		unit:       "requests/sec",
+		calc:       "derivative",
+	},
+	{
+		name:       "lustre_read_bw",
+		lineprefix: "read_bytes",
+		lineoffset: 6,
+		unit:       "bytes/sec",
+		calc:       "derivative",
+	},
+	{
+		name:       "lustre_write_bw",
+		lineprefix: "write_bytes",
+		lineoffset: 6,
+		unit:       "bytes/sec",
+		calc:       "derivative",
+	},
+}
+
 func (m *LustreCollector) Init(config json.RawMessage) error {
 	var err error
 	m.name = "LustreCollector"
@@ -92,42 +297,28 @@ func (m *LustreCollector) Init(config json.RawMessage) error {
 	m.setup()
 	m.tags = map[string]string{"type": "node"}
 	m.meta = map[string]string{"source": m.name, "group": "Lustre"}
-	defmatches := map[string]map[string]int{
-		"read_bytes":       {"lustre_read_bytes": 6, "lustre_read_requests": 1},
-		"write_bytes":      {"lustre_write_bytes": 6, "lustre_write_requests": 1},
-		"open":             {"lustre_open": 1},
-		"close":            {"lustre_close": 1},
-		"setattr":          {"lustre_setattr": 1},
-		"getattr":          {"lustre_getattr": 1},
-		"statfs":           {"lustre_statfs": 1},
-		"inode_permission": {"lustre_inode_permission": 1}}
 
 	// Lustre file system statistics can only be queried by user root
-	user, err := user.Current()
-	if err != nil {
-		cclog.ComponentError(m.name, "Failed to get current user:", err.Error())
-		return err
-	}
-	if user.Uid != "0" {
-		cclog.ComponentError(m.name, "Lustre file system statistics can only be queried by user root:", err.Error())
-		return err
+	// or with password-less sudo
+	if !m.config.Sudo {
+		user, err := user.Current()
+		if err != nil {
+			cclog.ComponentError(m.name, "Failed to get current user:", err.Error())
+			return err
+		}
+		if user.Uid != "0" {
+			cclog.ComponentError(m.name, "Lustre file system statistics can only be queried by user root")
+			return err
+		}
+	} else {
+		p, err := exec.LookPath("sudo")
+		if err != nil {
+			cclog.ComponentError(m.name, "Cannot find 'sudo'")
+			return err
+		}
+		m.sudoCmd = p
 	}
 
-	m.matches = make(map[string]map[string]int)
-	for lineprefix, names := range defmatches {
-		for metricname, offset := range names {
-			_, skip := stringArrayContains(m.config.ExcludeMetrics, metricname)
-			if skip {
-				continue
-			}
-			if _, prefixExist := m.matches[lineprefix]; !prefixExist {
-				m.matches[lineprefix] = make(map[string]int)
-			}
-			if _, metricExist := m.matches[lineprefix][metricname]; !metricExist {
-				m.matches[lineprefix][metricname] = offset
-			}
-		}
-	}
 	p, err := exec.LookPath(m.config.LCtlCommand)
 	if err != nil {
 		p, err = exec.LookPath(LCTL_CMD)
@@ -137,19 +328,50 @@ func (m *LustreCollector) Init(config json.RawMessage) error {
 	}
 	m.lctl = p
 
+	m.definitions = []LustreMetricDefinition{}
+	if m.config.SendAbsoluteValues {
+		for _, def := range LustreAbsMetrics {
+			if _, skip := stringArrayContains(m.config.ExcludeMetrics, def.name); !skip {
+				m.definitions = append(m.definitions, def)
+			}
+		}
+	}
+	if m.config.SendDiffValues {
+		for _, def := range LustreDiffMetrics {
+			if _, skip := stringArrayContains(m.config.ExcludeMetrics, def.name); !skip {
+				m.definitions = append(m.definitions, def)
+			}
+		}
+	}
+	if m.config.SendDerivedValues {
+		for _, def := range LustreDeriveMetrics {
+			if _, skip := stringArrayContains(m.config.ExcludeMetrics, def.name); !skip {
+				m.definitions = append(m.definitions, def)
+			}
+		}
+	}
+	if len(m.definitions) == 0 {
+		return errors.New("no metrics to collect")
+	}
+
 	devices := m.getDevices()
 	if len(devices) == 0 {
-		return errors.New("no metrics to collect")
+		return errors.New("no Lustre devices found")
 	}
 	m.stats = make(map[string]map[string]int64)
 	for _, d := range devices {
 		m.stats[d] = make(map[string]int64)
-		for _, names := range m.matches {
-			for metricname := range names {
-				m.stats[d][metricname] = 0
+		data := m.getDeviceDataCommand(d)
+		for _, def := range m.definitions {
+			x, err := getMetricData(data, def.lineprefix, def.lineoffset)
+			if err == nil {
+				m.stats[d][def.name] = x
+			} else {
+				m.stats[d][def.name] = 0
 			}
 		}
 	}
+	m.lastTimestamp = time.Now()
 	m.init = true
 	return nil
 }
@@ -158,54 +380,49 @@ func (m *LustreCollector) Read(interval time.Duration, output chan lp.CCMetric)
 	if !m.init {
 		return
 	}
+	now := time.Now()
+	tdiff := now.Sub(m.lastTimestamp)
 	for device, devData := range m.stats {
-		stats := m.getDeviceDataCommand(device)
-		processed := []string{}
-
-		for _, line := range stats {
-			lf := strings.Fields(line)
-			if len(lf) > 1 {
-				if fields, ok := m.matches[lf[0]]; ok {
-					for name, idx := range fields {
-						x, err := strconv.ParseInt(lf[idx], 0, 64)
-						if err != nil {
-							continue
-						}
-						value := x - devData[name]
-						devData[name] = x
-						if value < 0 {
-							value = 0
-						}
-						y, err := lp.New(name, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now())
-						if err == nil {
-							y.AddTag("device", device)
-							if strings.Contains(name, "byte") {
-								y.AddMeta("unit", "Byte")
-							}
-							output <- y
-							if m.config.SendAllMetrics {
-								processed = append(processed, name)
-							}
-						}
-					}
-				}
+		data := m.getDeviceDataCommand(device)
+		for _, def := range m.definitions {
+			var use_x int64
+			var err error
+			var y lp.CCMetric
+			x, err := getMetricData(data, def.lineprefix, def.lineoffset)
+			if err == nil {
+				use_x = x
+			} else {
+				use_x = devData[def.name]
 			}
-		}
-		if m.config.SendAllMetrics {
-			for name := range devData {
-				if _, done := stringArrayContains(processed, name); !done {
-					y, err := lp.New(name, m.tags, m.meta, map[string]interface{}{"value": 0}, time.Now())
-					if err == nil {
-						y.AddTag("device", device)
-						if strings.Contains(name, "byte") {
-							y.AddMeta("unit", "Byte")
-						}
-						output <- y
-					}
+			var value interface{}
+			switch def.calc {
+			case "none":
+				value = use_x
+				y, err = lp.New(def.name, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now())
+			case "difference":
+				value = use_x - devData[def.name]
+				if value.(int64) < 0 {
+					value = 0
 				}
+				y, err = lp.New(def.name, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now())
+			case "derivative":
+				value = float64(use_x-devData[def.name]) / tdiff.Seconds()
+				if value.(float64) < 0 {
+					value = 0
+				}
+				y, err = lp.New(def.name, m.tags, m.meta, map[string]interface{}{"value": value}, time.Now())
 			}
+			if err == nil {
+				y.AddTag("device", device)
+				if len(def.unit) > 0 {
+					y.AddMeta("unit", def.unit)
+				}
+				output <- y
+			}
+			devData[def.name] = use_x
 		}
 	}
+	m.lastTimestamp = now
 }
 
 func (m *LustreCollector) Close() {

collectors/lustreMetric.md

@@ -3,27 +3,44 @@
 
 ```json
   "lustrestat": {
-    "procfiles" : [
-      "/proc/fs/lustre/llite/lnec-XXXXXX/stats"
-    ],
+    "lctl_command": "/path/to/lctl",
     "exclude_metrics": [
       "setattr",
       "getattr"
-    ]
+    ],
+    "send_abs_values" : true,
+    "send_derived_values" : true,
+    "send_diff_values": true,
+    "use_sudo": false
   }
 ```
 
-The `lustrestat` collector reads from the procfs stat files for Lustre like `/proc/fs/lustre/llite/lnec-XXXXXX/stats`.
+The `lustrestat` collector uses the `lctl` application with the `get_param` option to get all `llite` metrics (Lustre client). The `llite` metrics are only available for root users. If password-less sudo is configured, you can enable `sudo` in the configuration.
 
 Metrics:
-* `read_bytes`
-* `read_requests`
-* `write_bytes`
-* `write_requests`
-* `open`
-* `close`
-* `getattr`
-* `setattr`
-* `statfs`
-* `inode_permission`
+* `lustre_read_bytes` (unit `bytes`)
+* `lustre_read_requests` (unit `requests`)
+* `lustre_write_bytes` (unit `bytes`)
+* `lustre_write_requests` (unit `requests`)
+* `lustre_open`
+* `lustre_close`
+* `lustre_getattr`
+* `lustre_setattr`
+* `lustre_statfs`
+* `lustre_inode_permission`
+* `lustre_read_bw` (if `send_derived_values == true`, unit `bytes/sec`)
+* `lustre_write_bw` (if `send_derived_values == true`, unit `bytes/sec`)
+* `lustre_read_requests_rate` (if `send_derived_values == true`, unit `requests/sec`)
+* `lustre_write_requests_rate` (if `send_derived_values == true`, unit `requests/sec`)
+* `lustre_read_bytes_diff` (if `send_diff_values == true`, unit `bytes`)
+* `lustre_read_requests_diff` (if `send_diff_values == true`, unit `requests`)
+* `lustre_write_bytes_diff` (if `send_diff_values == true`, unit `bytes`)
+* `lustre_write_requests_diff` (if `send_diff_values == true`, unit `requests`)
+* `lustre_open_diff` (if `send_diff_values == true`)
+* `lustre_close_diff` (if `send_diff_values == true`)
+* `lustre_getattr_diff` (if `send_diff_values == true`)
+* `lustre_setattr_diff` (if `send_diff_values == true`)
+* `lustre_statfs_diff` (if `send_diff_values == true`)
+* `lustre_inode_permission_diff` (if `send_diff_values == true`)
 
+This collector adds an `device` tag.

collectors/memstatMetric.go

@@ -1,46 +1,89 @@
 package collectors
 
 import (
+	"bufio"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
-	"log"
+	"os"
+	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
 )
 
-const MEMSTATFILE = `/proc/meminfo`
+const MEMSTATFILE = "/proc/meminfo"
+const NUMA_MEMSTAT_BASE = "/sys/devices/system/node"
 
 type MemstatCollectorConfig struct {
 	ExcludeMetrics []string `json:"exclude_metrics"`
+	NodeStats      bool     `json:"node_stats,omitempty"`
+	NumaStats      bool     `json:"numa_stats,omitempty"`
+}
+
+type MemstatCollectorNode struct {
+	file string
+	tags map[string]string
 }
 
 type MemstatCollector struct {
 	metricCollector
-	stats   map[string]int64
-	tags    map[string]string
-	matches map[string]string
-	config  MemstatCollectorConfig
+	stats       map[string]int64
+	tags        map[string]string
+	matches     map[string]string
+	config      MemstatCollectorConfig
+	nodefiles   map[int]MemstatCollectorNode
+	sendMemUsed bool
+}
+
+func getStats(filename string) map[string]float64 {
+	stats := make(map[string]float64)
+	file, err := os.Open(filename)
+	if err != nil {
+		cclog.Error(err.Error())
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		linefields := strings.Fields(line)
+		if len(linefields) == 3 {
+			v, err := strconv.ParseFloat(linefields[1], 64)
+			if err == nil {
+				stats[strings.Trim(linefields[0], ":")] = v
+			}
+		} else if len(linefields) == 5 {
+			v, err := strconv.ParseFloat(linefields[3], 64)
+			if err == nil {
+				stats[strings.Trim(linefields[0], ":")] = v
+			}
+		}
+	}
+	return stats
 }
 
 func (m *MemstatCollector) Init(config json.RawMessage) error {
 	var err error
 	m.name = "MemstatCollector"
+	m.config.NodeStats = true
+	m.config.NumaStats = false
 	if len(config) > 0 {
 		err = json.Unmarshal(config, &m.config)
 		if err != nil {
 			return err
 		}
 	}
-	m.meta = map[string]string{"source": m.name, "group": "Memory", "unit": "kByte"}
+	m.meta = map[string]string{"source": m.name, "group": "Memory", "unit": "GByte"}
 	m.stats = make(map[string]int64)
 	m.matches = make(map[string]string)
 	m.tags = map[string]string{"type": "node"}
-	matches := map[string]string{`MemTotal`: "mem_total",
+	matches := map[string]string{
+		"MemTotal":     "mem_total",
 		"SwapTotal":    "swap_total",
 		"SReclaimable": "mem_sreclaimable",
 		"Slab":         "mem_slab",
@@ -48,21 +91,58 @@ func (m *MemstatCollector) Init(config json.RawMessage) error {
 		"Buffers":      "mem_buffers",
 		"Cached":       "mem_cached",
 		"MemAvailable": "mem_available",
-		"SwapFree":     "swap_free"}
+		"SwapFree":     "swap_free",
+		"MemShared":    "mem_shared",
+	}
 	for k, v := range matches {
 		_, skip := stringArrayContains(m.config.ExcludeMetrics, k)
 		if !skip {
 			m.matches[k] = v
 		}
 	}
+	m.sendMemUsed = false
+	if _, skip := stringArrayContains(m.config.ExcludeMetrics, "mem_used"); !skip {
+		m.sendMemUsed = true
+	}
 	if len(m.matches) == 0 {
-		return errors.New("No metrics to collect")
+		return errors.New("no metrics to collect")
 	}
 	m.setup()
-	_, err = ioutil.ReadFile(string(MEMSTATFILE))
-	if err == nil {
-		m.init = true
+
+	if m.config.NodeStats {
+		if stats := getStats(MEMSTATFILE); len(stats) == 0 {
+			return fmt.Errorf("cannot read data from file %s", MEMSTATFILE)
+		}
 	}
+
+	if m.config.NumaStats {
+		globPattern := filepath.Join(NUMA_MEMSTAT_BASE, "node[0-9]*", "meminfo")
+		regex := regexp.MustCompile(filepath.Join(NUMA_MEMSTAT_BASE, "node(\\d+)", "meminfo"))
+		files, err := filepath.Glob(globPattern)
+		if err == nil {
+			m.nodefiles = make(map[int]MemstatCollectorNode)
+			for _, f := range files {
+				if stats := getStats(f); len(stats) == 0 {
+					return fmt.Errorf("cannot read data from file %s", f)
+				}
+				rematch := regex.FindStringSubmatch(f)
+				if len(rematch) == 2 {
+					id, err := strconv.Atoi(rematch[1])
+					if err == nil {
+						f := MemstatCollectorNode{
+							file: f,
+							tags: map[string]string{
+								"type":    "memoryDomain",
+								"type-id": fmt.Sprintf("%d", id),
+							},
+						}
+						m.nodefiles[id] = f
+					}
+				}
+			}
+		}
+	}
+	m.init = true
 	return err
 }
 
@@ -71,56 +151,44 @@ func (m *MemstatCollector) Read(interval time.Duration, output chan lp.CCMetric)
 		return
 	}
 
-	buffer, err := ioutil.ReadFile(string(MEMSTATFILE))
-	if err != nil {
-		log.Print(err)
-		return
-	}
-
-	ll := strings.Split(string(buffer), "\n")
-	for _, line := range ll {
-		ls := strings.Split(line, `:`)
-		if len(ls) > 1 {
-			lv := strings.Fields(ls[1])
-			m.stats[ls[0]], err = strconv.ParseInt(lv[0], 0, 64)
+	sendStats := func(stats map[string]float64, tags map[string]string) {
+		for match, name := range m.matches {
+			var value float64 = 0
+			if v, ok := stats[match]; ok {
+				value = v
+			}
+			y, err := lp.New(name, tags, m.meta, map[string]interface{}{"value": value * 1e-6}, time.Now())
+			if err == nil {
+				output <- y
+			}
 		}
-	}
-
-	if _, exists := m.stats[`MemTotal`]; !exists {
-		err = errors.New("Parse error")
-		log.Print(err)
-		return
-	}
-
-	for match, name := range m.matches {
-		if _, exists := m.stats[match]; !exists {
-			err = fmt.Errorf("Parse error for %s : %s", match, name)
-			log.Print(err)
-			continue
-		}
-		y, err := lp.New(name, m.tags, m.meta, map[string]interface{}{"value": int(float64(m.stats[match]) * 1.0e-3)}, time.Now())
-		if err == nil {
-			output <- y
-		}
-	}
-
-	if _, free := m.stats[`MemFree`]; free {
-		if _, buffers := m.stats[`Buffers`]; buffers {
-			if _, cached := m.stats[`Cached`]; cached {
-				memUsed := m.stats[`MemTotal`] - (m.stats[`MemFree`] + m.stats[`Buffers`] + m.stats[`Cached`])
-				_, skip := stringArrayContains(m.config.ExcludeMetrics, "mem_used")
-				y, err := lp.New("mem_used", m.tags, m.meta, map[string]interface{}{"value": int(float64(memUsed) * 1.0e-3)}, time.Now())
-				if err == nil && !skip {
-					output <- y
+		if m.sendMemUsed {
+			memUsed := 0.0
+			if totalVal, total := stats["MemTotal"]; total {
+				if freeVal, free := stats["MemFree"]; free {
+					if bufVal, buffers := stats["Buffers"]; buffers {
+						if cacheVal, cached := stats["Cached"]; cached {
+							memUsed = totalVal - (freeVal + bufVal + cacheVal)
+						}
+					}
 				}
 			}
+			y, err := lp.New("mem_used", tags, m.meta, map[string]interface{}{"value": memUsed * 1e-6}, time.Now())
+			if err == nil {
+				output <- y
+			}
 		}
 	}
-	if _, found := m.stats[`MemShared`]; found {
-		_, skip := stringArrayContains(m.config.ExcludeMetrics, "mem_shared")
-		y, err := lp.New("mem_shared", m.tags, m.meta, map[string]interface{}{"value": int(float64(m.stats[`MemShared`]) * 1.0e-3)}, time.Now())
-		if err == nil && !skip {
-			output <- y
+
+	if m.config.NodeStats {
+		nodestats := getStats(MEMSTATFILE)
+		sendStats(nodestats, m.tags)
+	}
+
+	if m.config.NumaStats {
+		for _, nodeConf := range m.nodefiles {
+			stats := getStats(nodeConf.file)
+			sendStats(stats, nodeConf.tags)
 		}
 	}
 }

collectors/metricCollector.go

@@ -13,29 +13,30 @@ import (
 )
 
 type MetricCollector interface {
-	Name() string
-	Init(config json.RawMessage) error
-	Initialized() bool
-	Read(duration time.Duration, output chan lp.CCMetric)
-	Close()
+	Name() string                                         // Name of the metric collector
+	Init(config json.RawMessage) error                    // Initialize metric collector
+	Initialized() bool                                    // Is metric collector initialized?
+	Read(duration time.Duration, output chan lp.CCMetric) // Read metrics from metric collector
+	Close()                                               // Close / finish metric collector
 }
 
 type metricCollector struct {
-	name string
-	init bool
-	meta map[string]string
+	name string            // name of the metric
+	init bool              // is metric collector initialized?
+	meta map[string]string // static meta data tags
 }
 
-// Name() returns the name of the metric collector
+// Name returns the name of the metric collector
 func (c *metricCollector) Name() string {
 	return c.name
 }
 
+// Setup is for future use
 func (c *metricCollector) setup() error {
 	return nil
 }
 
-// Initialized() indicates whether the metric collector has been initialized.
+// Initialized indicates whether the metric collector has been initialized
 func (c *metricCollector) Initialized() bool {
 	return c.init
 }
@@ -64,6 +65,7 @@ func stringArrayContains(array []string, str string) (int, bool) {
 	return -1, false
 }
 
+// SocketList returns the list of physical sockets as read from /proc/cpuinfo
 func SocketList() []int {
 	buffer, err := ioutil.ReadFile("/proc/cpuinfo")
 	if err != nil {
@@ -89,6 +91,7 @@ func SocketList() []int {
 	return packs
 }
 
+// CpuList returns the list of physical CPUs (in contrast to logical CPUs) as read from /proc/cpuinfo
 func CpuList() []int {
 	buffer, err := ioutil.ReadFile("/proc/cpuinfo")
 	if err != nil {
@@ -117,10 +120,10 @@ func CpuList() []int {
 // RemoveFromStringList removes the string r from the array of strings s
 // If r is not contained in the array an error is returned
 func RemoveFromStringList(s []string, r string) ([]string, error) {
-	for i, item := range s {
-		if r == item {
+	for i := range s {
+		if r == s[i] {
 			return append(s[:i], s[i+1:]...), nil
 		}
 	}
-	return s, fmt.Errorf("No such string in list")
+	return s, fmt.Errorf("no such string in list")
 }

collectors/netstatMetric.go

@@ -13,22 +13,27 @@ import (
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
 )
 
-const NETSTATFILE = `/proc/net/dev`
+const NETSTATFILE = "/proc/net/dev"
 
 type NetstatCollectorConfig struct {
-	IncludeDevices []string `json:"include_devices"`
+	IncludeDevices     []string `json:"include_devices"`
+	SendAbsoluteValues bool     `json:"send_abs_values"`
+	SendDerivedValues  bool     `json:"send_derived_values"`
 }
 
 type NetstatCollectorMetric struct {
-	index     int
-	lastValue float64
+	name       string
+	index      int
+	tags       map[string]string
+	meta       map[string]string
+	meta_rates map[string]string
+	lastValue  int64
 }
 
 type NetstatCollector struct {
 	metricCollector
 	config        NetstatCollectorConfig
-	matches       map[string]map[string]NetstatCollectorMetric
-	devtags       map[string]map[string]string
+	matches       map[string][]NetstatCollectorMetric
 	lastTimestamp time.Time
 }
 
@@ -36,15 +41,33 @@ func (m *NetstatCollector) Init(config json.RawMessage) error {
 	m.name = "NetstatCollector"
 	m.setup()
 	m.lastTimestamp = time.Now()
-	m.meta = map[string]string{"source": m.name, "group": "Network"}
-	m.devtags = make(map[string]map[string]string)
-	nameIndexMap := map[string]int{
-		"net_bytes_in":  1,
-		"net_pkts_in":   2,
-		"net_bytes_out": 9,
-		"net_pkts_out":  10,
-	}
-	m.matches = make(map[string]map[string]NetstatCollectorMetric)
+
+	const (
+		fieldInterface = iota
+		fieldReceiveBytes
+		fieldReceivePackets
+		fieldReceiveErrs
+		fieldReceiveDrop
+		fieldReceiveFifo
+		fieldReceiveFrame
+		fieldReceiveCompressed
+		fieldReceiveMulticast
+		fieldTransmitBytes
+		fieldTransmitPackets
+		fieldTransmitErrs
+		fieldTransmitDrop
+		fieldTransmitFifo
+		fieldTransmitColls
+		fieldTransmitCarrier
+		fieldTransmitCompressed
+	)
+
+	m.matches = make(map[string][]NetstatCollectorMetric)
+
+	// Set default configuration,
+	m.config.SendAbsoluteValues = true
+	m.config.SendDerivedValues = false
+	// Read configuration file, allow overwriting default config
 	if len(config) > 0 {
 		err := json.Unmarshal(config, &m.config)
 		if err != nil {
@@ -52,7 +75,9 @@ func (m *NetstatCollector) Init(config json.RawMessage) error {
 			return err
 		}
 	}
-	file, err := os.Open(string(NETSTATFILE))
+
+	// Check access to net statistic file
+	file, err := os.Open(NETSTATFILE)
 	if err != nil {
 		cclog.ComponentError(m.name, err.Error())
 		return err
@@ -62,23 +87,65 @@ func (m *NetstatCollector) Init(config json.RawMessage) error {
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		l := scanner.Text()
+
+		// Skip lines with no net device entry
 		if !strings.Contains(l, ":") {
 			continue
 		}
+
+		// Split line into fields
 		f := strings.Fields(l)
+
+		// Get net device entry
 		dev := strings.Trim(f[0], ": ")
+
+		// Check if device is a included device
 		if _, ok := stringArrayContains(m.config.IncludeDevices, dev); ok {
-			m.matches[dev] = make(map[string]NetstatCollectorMetric)
-			for name, idx := range nameIndexMap {
-				m.matches[dev][name] = NetstatCollectorMetric{
-					index:     idx,
-					lastValue: 0,
-				}
+			tags := map[string]string{"device": dev, "type": "node"}
+			meta_unit_byte := map[string]string{"source": m.name, "group": "Network", "unit": "bytes"}
+			meta_unit_byte_per_sec := map[string]string{"source": m.name, "group": "Network", "unit": "bytes/sec"}
+			meta_unit_pkts := map[string]string{"source": m.name, "group": "Network", "unit": "packets"}
+			meta_unit_pkts_per_sec := map[string]string{"source": m.name, "group": "Network", "unit": "packets/sec"}
+
+			m.matches[dev] = []NetstatCollectorMetric{
+				{
+					name:       "net_bytes_in",
+					index:      fieldReceiveBytes,
+					lastValue:  -1,
+					tags:       tags,
+					meta:       meta_unit_byte,
+					meta_rates: meta_unit_byte_per_sec,
+				},
+				{
+					name:       "net_pkts_in",
+					index:      fieldReceivePackets,
+					lastValue:  -1,
+					tags:       tags,
+					meta:       meta_unit_pkts,
+					meta_rates: meta_unit_pkts_per_sec,
+				},
+				{
+					name:       "net_bytes_out",
+					index:      fieldTransmitBytes,
+					lastValue:  -1,
+					tags:       tags,
+					meta:       meta_unit_byte,
+					meta_rates: meta_unit_byte_per_sec,
+				},
+				{
+					name:       "net_pkts_out",
+					index:      fieldTransmitPackets,
+					lastValue:  -1,
+					tags:       tags,
+					meta:       meta_unit_pkts,
+					meta_rates: meta_unit_pkts_per_sec,
+				},
 			}
-			m.devtags[dev] = map[string]string{"device": dev, "type": "node"}
 		}
+
 	}
-	if len(m.devtags) == 0 {
+
+	if len(m.matches) == 0 {
 		return errors.New("no devices to collector metrics found")
 	}
 	m.init = true
@@ -89,50 +156,62 @@ func (m *NetstatCollector) Read(interval time.Duration, output chan lp.CCMetric)
 	if !m.init {
 		return
 	}
+	// Current time stamp
 	now := time.Now()
+	// time difference to last time stamp
+	timeDiff := now.Sub(m.lastTimestamp).Seconds()
+	// Save current timestamp
+	m.lastTimestamp = now
+
 	file, err := os.Open(string(NETSTATFILE))
 	if err != nil {
 		cclog.ComponentError(m.name, err.Error())
 		return
 	}
 	defer file.Close()
-	tdiff := now.Sub(m.lastTimestamp)
 
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		l := scanner.Text()
+
+		// Skip lines with no net device entry
 		if !strings.Contains(l, ":") {
 			continue
 		}
+
+		// Split line into fields
 		f := strings.Fields(l)
+
+		// Get net device entry
 		dev := strings.Trim(f[0], ":")
 
+		// Check if device is a included device
 		if devmetrics, ok := m.matches[dev]; ok {
-			for name, data := range devmetrics {
-				v, err := strconv.ParseFloat(f[data.index], 64)
-				if err == nil {
-					vdiff := v - data.lastValue
-					value := vdiff / tdiff.Seconds()
-					if data.lastValue == 0 {
-						value = 0
-					}
-					data.lastValue = v
-					y, err := lp.New(name, m.devtags[dev], m.meta, map[string]interface{}{"value": value}, now)
-					if err == nil {
-						switch {
-						case strings.Contains(name, "byte"):
-							y.AddMeta("unit", "bytes/sec")
-						case strings.Contains(name, "pkt"):
-							y.AddMeta("unit", "packets/sec")
-						}
+			for i := range devmetrics {
+				metric := &devmetrics[i]
+
+				// Read value
+				v, err := strconv.ParseInt(f[metric.index], 10, 64)
+				if err != nil {
+					continue
+				}
+				if m.config.SendAbsoluteValues {
+					if y, err := lp.New(metric.name, metric.tags, metric.meta, map[string]interface{}{"value": v}, now); err == nil {
 						output <- y
 					}
-					devmetrics[name] = data
+				}
+				if m.config.SendDerivedValues {
+					if metric.lastValue >= 0 {
+						rate := float64(v-metric.lastValue) / timeDiff
+						if y, err := lp.New(metric.name+"_bw", metric.tags, metric.meta_rates, map[string]interface{}{"value": rate}, now); err == nil {
+							output <- y
+						}
+					}
+					metric.lastValue = v
 				}
 			}
 		}
 	}
-	m.lastTimestamp = time.Now()
 }
 
 func (m *NetstatCollector) Close() {

collectors/netstatMetric.md

@@ -5,17 +5,23 @@
   "netstat": {
     "include_devices": [
       "eth0"
-    ]
+    ],
+    "send_abs_values" : true,
+    "send_derived_values" : true
   }
 ```
 
 The `netstat` collector reads data from `/proc/net/dev` and outputs a handful **node** metrics. With the `include_devices` list you can specify which network devices should be measured. **Note**: Most other collectors use an _exclude_ list instead of an include list.
 
 Metrics:
-* `net_bytes_in` (`unit=bytes/sec`)
-* `net_bytes_out` (`unit=bytes/sec`)
-* `net_pkts_in` (`unit=packets/sec`)
-* `net_pkts_out` (`unit=packets/sec`)
+* `net_bytes_in` (`unit=bytes`)
+* `net_bytes_out` (`unit=bytes`)
+* `net_pkts_in` (`unit=packets`)
+* `net_pkts_out` (`unit=packets`)
+* `net_bytes_in_bw` (`unit=bytes/sec` if `send_derived_values == true`)
+* `net_bytes_out_bw` (`unit=bytes/sec` if `send_derived_values == true`)
+* `net_pkts_in_bw` (`unit=packets/sec` if `send_derived_values == true`)
+* `net_pkts_out_bw` (`unit=packets/sec` if `send_derived_values == true`)
 
 The device name is added as tag `device`.
 

collectors/sampleMetric.go

@@ -0,0 +1,92 @@
+package collectors
+
+import (
+	"encoding/json"
+	"time"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+)
+
+// These are the fields we read from the JSON configuration
+type SampleCollectorConfig struct {
+	Interval string `json:"interval"`
+}
+
+// This contains all variables we need during execution and the variables
+// defined by metricCollector (name, init, ...)
+type SampleCollector struct {
+	metricCollector
+	config SampleTimerCollectorConfig // the configuration structure
+	meta   map[string]string          // default meta information
+	tags   map[string]string          // default tags
+}
+
+// Functions to implement MetricCollector interface
+// Init(...), Read(...), Close()
+// See: metricCollector.go
+
+// Init initializes the sample collector
+// Called once by the collector manager
+// All tags, meta data tags and metrics that do not change over the runtime should be set here
+func (m *SampleCollector) Init(config json.RawMessage) error {
+	var err error = nil
+	// Always set the name early in Init() to use it in cclog.Component* functions
+	m.name = "InternalCollector"
+	// This is for later use, also call it early
+	m.setup()
+	// Define meta information sent with each metric
+	// (Can also be dynamic or this is the basic set with extension through AddMeta())
+	m.meta = map[string]string{"source": m.name, "group": "SAMPLE"}
+	// Define tags sent with each metric
+	// The 'type' tag is always needed, it defines the granulatity of the metric
+	// node -> whole system
+	// socket -> CPU socket (requires socket ID as 'type-id' tag)
+	// cpu -> single CPU hardware thread (requires cpu ID as 'type-id' tag)
+	m.tags = map[string]string{"type": "node"}
+	// Read in the JSON configuration
+	if len(config) > 0 {
+		err = json.Unmarshal(config, &m.config)
+		if err != nil {
+			cclog.ComponentError(m.name, "Error reading config:", err.Error())
+			return err
+		}
+	}
+
+	// Set up everything that the collector requires during the Read() execution
+	// Check files required, test execution of some commands, create data structure
+	// for all topological entities (sockets, NUMA domains, ...)
+	// Return some useful error message in case of any failures
+
+	// Set this flag only if everything is initialized properly, all required files exist, ...
+	m.init = true
+	return err
+}
+
+// Read collects all metrics belonging to the sample collector
+// and sends them through the output channel to the collector manager
+func (m *SampleCollector) Read(interval time.Duration, output chan lp.CCMetric) {
+	// Create a sample metric
+	timestamp := time.Now()
+
+	value := 1.0
+	// If you want to measure something for a specific amount of time, use interval
+	// start := readState()
+	// time.Sleep(interval)
+	// stop := readState()
+	// value = (stop - start) / interval.Seconds()
+
+	y, err := lp.New("sample_metric", m.tags, m.meta, map[string]interface{}{"value": value}, timestamp)
+	if err == nil {
+		// Send it to output channel
+		output <- y
+	}
+
+}
+
+// Close metric collector: close network connection, close files, close libraries, ...
+// Called once by the collector manager
+func (m *SampleCollector) Close() {
+	// Unset flag
+	m.init = false
+}

collectors/sampleTimerMetric.go

@@ -0,0 +1,122 @@
+package collectors
+
+import (
+	"encoding/json"
+	"sync"
+	"time"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+)
+
+// These are the fields we read from the JSON configuration
+type SampleTimerCollectorConfig struct {
+	Interval string `json:"interval"`
+}
+
+// This contains all variables we need during execution and the variables
+// defined by metricCollector (name, init, ...)
+type SampleTimerCollector struct {
+	metricCollector
+	wg       sync.WaitGroup             // sync group for management
+	done     chan bool                  // channel for management
+	meta     map[string]string          // default meta information
+	tags     map[string]string          // default tags
+	config   SampleTimerCollectorConfig // the configuration structure
+	interval time.Duration              // the interval parsed from configuration
+	ticker   *time.Ticker               // own timer
+	output   chan lp.CCMetric           // own internal output channel
+}
+
+func (m *SampleTimerCollector) Init(name string, config json.RawMessage) error {
+	var err error = nil
+	// Always set the name early in Init() to use it in cclog.Component* functions
+	m.name = "SampleTimerCollector"
+	// This is for later use, also call it early
+	m.setup()
+	// Define meta information sent with each metric
+	// (Can also be dynamic or this is the basic set with extension through AddMeta())
+	m.meta = map[string]string{"source": m.name, "group": "SAMPLE"}
+	// Define tags sent with each metric
+	// The 'type' tag is always needed, it defines the granulatity of the metric
+	// node -> whole system
+	// socket -> CPU socket (requires socket ID as 'type-id' tag)
+	// cpu -> single CPU hardware thread (requires cpu ID as 'type-id' tag)
+	m.tags = map[string]string{"type": "node"}
+	// Read in the JSON configuration
+	if len(config) > 0 {
+		err = json.Unmarshal(config, &m.config)
+		if err != nil {
+			cclog.ComponentError(m.name, "Error reading config:", err.Error())
+			return err
+		}
+	}
+	// Parse the read interval duration
+	m.interval, err = time.ParseDuration(m.config.Interval)
+	if err != nil {
+		cclog.ComponentError(m.name, "Error parsing interval:", err.Error())
+		return err
+	}
+
+	// Storage for output channel
+	m.output = nil
+	// Mangement channel for the timer function.
+	m.done = make(chan bool)
+	// Create the own ticker
+	m.ticker = time.NewTicker(m.interval)
+
+	// Start the timer loop with return functionality by sending 'true' to the done channel
+	m.wg.Add(1)
+	go func() {
+		select {
+		case <-m.done:
+			// Exit the timer loop
+			cclog.ComponentDebug(m.name, "Closing...")
+			m.wg.Done()
+			return
+		case timestamp := <-m.ticker.C:
+			// This is executed every timer tick but we have to wait until the first
+			// Read() to get the output channel
+			if m.output != nil {
+				m.ReadMetrics(timestamp)
+			}
+		}
+	}()
+
+	// Set this flag only if everything is initialized properly, all required files exist, ...
+	m.init = true
+	return err
+}
+
+// This function is called at each interval timer tick
+func (m *SampleTimerCollector) ReadMetrics(timestamp time.Time) {
+	// Create a sample metric
+
+	value := 1.0
+
+	// If you want to measure something for a specific amout of time, use interval
+	// start := readState()
+	// time.Sleep(interval)
+	// stop := readState()
+	// value = (stop - start) / interval.Seconds()
+
+	y, err := lp.New("sample_metric", m.tags, m.meta, map[string]interface{}{"value": value}, timestamp)
+	if err == nil && m.output != nil {
+		// Send it to output channel if we have a valid channel
+		m.output <- y
+	}
+}
+
+func (m *SampleTimerCollector) Read(interval time.Duration, output chan lp.CCMetric) {
+	// Capture output channel
+	m.output = output
+}
+
+func (m *SampleTimerCollector) Close() {
+	// Send signal to the timer loop to stop it
+	m.done <- true
+	// Wait until the timer loop is done
+	m.wg.Wait()
+	// Unset flag
+	m.init = false
+}

collectors/tempMetric.go

@@ -70,10 +70,10 @@ func (m *TempCollector) Init(config json.RawMessage) error {
 	globPattern := filepath.Join("/sys/class/hwmon", "*", "temp*_input")
 	inputFiles, err := filepath.Glob(globPattern)
 	if err != nil {
-		return fmt.Errorf("Unable to glob files with pattern '%s': %v", globPattern, err)
+		return fmt.Errorf("unable to glob files with pattern '%s': %v", globPattern, err)
 	}
 	if inputFiles == nil {
-		return fmt.Errorf("Unable to find any files with pattern '%s'", globPattern)
+		return fmt.Errorf("unable to find any files with pattern '%s'", globPattern)
 	}
 
 	// Get sensor name for each temperature sensor file
@@ -158,7 +158,7 @@ func (m *TempCollector) Init(config json.RawMessage) error {
 
 	// Empty sensors map
 	if len(m.sensors) == 0 {
-		return fmt.Errorf("No temperature sensors found")
+		return fmt.Errorf("no temperature sensors found")
 	}
 
 	// Finished initialization

collectors/topprocsMetric.go

@@ -39,14 +39,14 @@ func (m *TopProcsCollector) Init(config json.RawMessage) error {
 		m.config.Num_procs = int(DEFAULT_NUM_PROCS)
 	}
 	if m.config.Num_procs <= 0 || m.config.Num_procs > MAX_NUM_PROCS {
-		return errors.New(fmt.Sprintf("num_procs option must be set in 'topprocs' config (range: 1-%d)", MAX_NUM_PROCS))
+		return fmt.Errorf("num_procs option must be set in 'topprocs' config (range: 1-%d)", MAX_NUM_PROCS)
 	}
 	m.setup()
 	command := exec.Command("ps", "-Ao", "comm", "--sort=-pcpu")
 	command.Wait()
 	_, err = command.Output()
 	if err != nil {
-		return errors.New("Failed to execute command")
+		return errors.New("failed to execute command")
 	}
 	m.init = true
 	return nil

docs/configuration.md

@@ -0,0 +1,189 @@
+# Configuring the CC metric collector
+
+The configuration of the CC metric collector consists of five configuration files: one global file and four component related files.
+
+## Global configuration
+
+The global file contains the paths to the other four files and some global options. 
+
+```json
+{
+  "sinks": "sinks.json",
+  "collectors" : "collectors.json",
+  "receivers" : "receivers.json",
+  "router" : "router.json",
+  "interval": 10,
+  "duration": 1
+}
+```
+
+Be aware that the paths are relative to the execution folder of the cc-metric-collector binary, so it is recommended to use absolute paths.
+
+## Component configuration
+
+The others are mainly list of of subcomponents: the collectors, the receivers, the router and the sinks. Their role is best shown in a picture:
+
+```mermaid
+flowchart LR
+
+  subgraph col ["Collectors"]
+  direction TB
+  cpustat["cpustat"]
+  memstat["memstat"]
+  tempstat["tempstat"]
+  misc["..."]
+  end
+  
+  subgraph Receivers ["Receivers"]
+  direction TB
+  nats["NATS"]
+  httprecv["HTTP"]
+  miscrecv[...]
+  end
+
+  subgraph calc["Aggregator"]
+  direction LR
+  cache["Cache"]
+  agg["Calculator"]
+  end
+
+  subgraph sinks ["Sinks"]
+  direction RL
+  influx["InfluxDB"]
+  ganglia["Ganglia"]
+  logger["Logfile"]
+  miscsink["..."]
+  end
+
+  cpustat --> CollectorManager["CollectorManager"]
+  memstat --> CollectorManager
+  tempstat --> CollectorManager
+  misc --> CollectorManager
+
+  nats  --> ReceiverManager["ReceiverManager"]
+  httprecv --> ReceiverManager
+  miscrecv --> ReceiverManager
+
+  CollectorManager --> newrouter["Router"]
+  ReceiverManager -.-> newrouter
+  calc -.-> newrouter
+  newrouter --> SinkManager["SinkManager"]
+  newrouter -.-> calc
+
+  SinkManager --> influx
+  SinkManager --> ganglia
+  SinkManager --> logger
+  SinkManager --> miscsink
+
+
+```
+
+There are four parts:
+- The collectors read data from files, execute commands and call dynamically loaded library function and send it to the router
+- The router can process metrics by cacheing and evaluating functions and conditions on them
+- The sinks send the metrics to storage backends
+- The receivers can be used to receive metrics from other collectors and forward them to the router. They can be used to create a tree-like structure of collectors.
+
+(A maybe better differentiation between collectors and receivers is that the collectors are called periodically while the receivers have their own logic and submit metrics at any time)
+
+
+### Collectors configuration file
+
+The collectors configuration file tells which metrics should be queried from the system. The metric gathering is logically grouped in so called 'Collectors'. So there are Collectors to read CPU, memory or filesystem statistics. The collectors configuration file is a list of these collectors with collector-specific configurations:
+
+```json
+{
+  "cpustat" : {},
+  "diskstat": {
+    "exclude_metrics": [
+      "disk_total"
+    ]
+  }
+}
+```
+
+The first one is the CPU statistics collector without any collector-specific setting. The second one enables disk mount statistics but excludes the metric `disk_total`.
+
+All names and possible collector-specific configuration options can be found [here](../collectors/README.md).
+
+Some collectors might dynamically load shared libraries. In order to enable these collectors, make sure that the shared library path is part of the `LD_LIBRARY_PATH` environment variable.
+
+### Sinks configuration file
+
+The sinks define the output/sending of metrics. The metrics can be forwarded to multiple sinks, even to sinks of the same type. The sinks configuration file is a list of these sinks, each with an individual name.
+
+```json
+{
+  "myinflux" : {
+    "type" : "influxasync",
+    "host": "localhost",
+    "port": "8086",
+    "organization" : "testorga",
+    "database" : "testbucket",
+    "password" : "<my secret JWT>"
+  },
+  "companyinflux" : {
+    "type" : "influxasync",
+    "host": "companyhost",
+    "port": "8086",
+    "organization" : "company",
+    "database" : "main",
+    "password" : "<company's secret JWT>"
+  }
+}
+```
+
+The above example configuration file defines two sink, both ot type `influxasync`. They are differentiated internally by the names: `myinflux` and `companyinflux`.
+
+All types and possible sink-specific configuration options can be found [here](../sinks/README.md).
+
+Some sinks might dynamically load shared libraries. In order to enable these sinks, make sure that the shared library path is part of the `LD_LIBRARY_PATH` environment variable.
+
+### Router configuration file
+
+The collectors and the sinks are connected through the router. The router forwards the metrics to the sinks but enables some data processing. A common example is to tag all passing metrics like adding `cluster=mycluster`. But also aggregations like "take the average of all 'ipc' metrics" (ipc -> Instructions Per Cycle). Since the configurations of these aggregations can be quite complicated, we refer to the router's [README](../internal/metricRouter/README.md).
+
+A simple router configuration file to start with looks like this:
+
+```json
+{
+    "add_tags" : [
+        {
+            "key" : "cluster",
+            "value" : "mycluster",
+            "if" : "*"
+        }
+    ],
+    "interval_timestamp" : false,
+    "num_cache_intervals" : 0
+}
+```
+
+With the `add_tags` section, we tell to attach the `cluster=mycluster` tag to each (`*` metric). The `interval_timestamp` tell the router to not touch the timestamp of metrics. It is possible to send all metrics within an interval with a common time stamp to avoid later alignment issues. The `num_cache_intervals` diables the cache completely. The cache is only required if you want to do complex metric aggregations.
+
+All configuration options can be found [here](../internal/metricRouter/README.md).
+
+### Receivers configuration file
+
+The receivers are a special feature of the CC Metric Collector to enable simpler integration into exising setups. While collectors query data from the local system, the receivers commonly get data from other systems through some network technology like HTTP or NATS. The idea is keep the current setup but send it to a CC Metric Collector which forwards it to the the destination system (if a sink exists for it). For most setups, the receivers are not required and an the receiver config file should contain only an empty JSON map (`{}`). 
+
+```json
+{
+  "nats_rack0": {
+    "type": "nats",
+    "address" : "nats-server.example.org",
+    "port" : "4222",
+    "subject" : "rack0",
+  },
+  "nats_rack1": {
+    "type": "nats",
+    "address" : "nats-server.example.org",
+    "port" : "4222",
+    "subject" : "rack1",
+  }
+}
+```
+
+This example configuration creates two receivers with the names `nats_rack0` and `nats_rack1`. While one subscribes to metrics published with the `rack0` subject, the other one subscribes to the `rack0` subject. The NATS server is the same as it manages all subjects in a subnet. (As example, the router could add tags `rack=0` and `rack=1` respectively to the received metrics.)
+
+All types and possible receiver-specific configuration options can be found [here](../receivers/README.md).

internal/ccMetric/ccMetric.go

@@ -24,8 +24,8 @@ type ccMetric struct {
 
 // ccMetric access functions
 type CCMetric interface {
-	ToPoint(metaAsTags bool) *write.Point  // Generate influxDB point for data type ccMetric
-	ToLineProtocol(metaAsTags bool) string // Generate influxDB line protocol for data type ccMetric
+	ToPoint(metaAsTags map[string]bool) *write.Point  // Generate influxDB point for data type ccMetric
+	ToLineProtocol(metaAsTags map[string]bool) string // Generate influxDB line protocol for data type ccMetric
 
 	Name() string        // Get metric name
 	SetName(name string) // Set metric name
@@ -61,25 +61,18 @@ func (m *ccMetric) String() string {
 }
 
 // ToLineProtocol generates influxDB line protocol for data type ccMetric
-func (m *ccMetric) ToPoint(metaAsTags bool) (p *write.Point) {
-
-	if !metaAsTags {
-		p = influxdb2.NewPoint(m.name, m.tags, m.fields, m.tm)
-	} else {
-		tags := make(map[string]string, len(m.tags)+len(m.meta))
-		for key, value := range m.tags {
-			tags[key] = value
+func (m *ccMetric) ToPoint(metaAsTags map[string]bool) (p *write.Point) {
+	p = influxdb2.NewPoint(m.name, m.tags, m.fields, m.tm)
+	for key, ok1 := range metaAsTags {
+		if val, ok2 := m.GetMeta(key); ok1 && ok2 {
+			p.AddTag(key, val)
 		}
-		for key, value := range m.meta {
-			tags[key] = value
-		}
-		p = influxdb2.NewPoint(m.name, tags, m.fields, m.tm)
 	}
-	return
+	return p
 }
 
 // ToLineProtocol generates influxDB line protocol for data type ccMetric
-func (m *ccMetric) ToLineProtocol(metaAsTags bool) string {
+func (m *ccMetric) ToLineProtocol(metaAsTags map[string]bool) string {
 
 	return write.PointToLineProtocol(
 		m.ToPoint(metaAsTags),

internal/metricRouter/metricRouter.go

@@ -25,6 +25,7 @@ type metricRouterTagConfig struct {
 
 // Metric router configuration
 type metricRouterConfig struct {
+	HostnameTagName   string                               `json:"hostname_tag"`        // Key name used when adding the hostname to a metric (default 'hostname')
 	AddTags           []metricRouterTagConfig              `json:"add_tags"`            // List of tags that are added when the condition is met
 	DelTags           []metricRouterTagConfig              `json:"delete_tags"`         // List of tags that are removed when the condition is met
 	IntervalAgg       []agg.MetricAggregatorIntervalConfig `json:"interval_aggregates"` // List of aggregation function processed at the end of an interval
@@ -33,6 +34,7 @@ type metricRouterConfig struct {
 	RenameMetrics     map[string]string                    `json:"rename_metrics"`      // Map to rename metric name from key to value
 	IntervalStamp     bool                                 `json:"interval_timestamp"`  // Update timestamp periodically by ticker each interval?
 	NumCacheIntervals int                                  `json:"num_cache_intervals"` // Number of intervals of cached metrics for evaluation
+	MaxForward        int                                  `json:"max_forward"`         // Number of maximal forwarded metrics at one select
 	dropMetrics       map[string]bool                      // Internal map for O(1) lookup
 }
 
@@ -76,7 +78,8 @@ func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, rout
 	r.cache_input = make(chan lp.CCMetric)
 	r.wg = wg
 	r.ticker = ticker
-	r.maxForward = ROUTER_MAX_FORWARD
+	r.config.MaxForward = ROUTER_MAX_FORWARD
+	r.config.HostnameTagName = "hostname"
 
 	// Set hostname
 	hostname, err := os.Hostname()
@@ -100,6 +103,7 @@ func (r *metricRouter) Init(ticker mct.MultiChanTicker, wg *sync.WaitGroup, rout
 		cclog.ComponentError("MetricRouter", err.Error())
 		return err
 	}
+	r.maxForward = r.config.MaxForward
 	if r.config.NumCacheIntervals > 0 {
 		r.cache, err = NewCache(r.cache_input, r.ticker, &r.cachewg, r.config.NumCacheIntervals)
 		if err != nil {
@@ -244,8 +248,10 @@ func (r *metricRouter) Start() {
 		cclog.ComponentDebug("MetricRouter", "FORWARD", point)
 		r.DoAddTags(point)
 		r.DoDelTags(point)
-		if new, ok := r.config.RenameMetrics[point.Name()]; ok {
+		name := point.Name()
+		if new, ok := r.config.RenameMetrics[name]; ok {
 			point.SetName(new)
+			point.AddMeta("oldname", name)
 		}
 		r.DoAddTags(point)
 		r.DoDelTags(point)
@@ -258,7 +264,7 @@ func (r *metricRouter) Start() {
 	// Foward message received from collector channel
 	coll_forward := func(p lp.CCMetric) {
 		// receive from metric collector
-		p.AddTag("hostname", r.hostname)
+		p.AddTag(r.config.HostnameTagName, r.hostname)
 		if r.config.IntervalStamp {
 			p.SetTime(r.timestamp)
 		}
@@ -287,7 +293,7 @@ func (r *metricRouter) Start() {
 	cache_forward := func(p lp.CCMetric) {
 		// receive from metric collector
 		if !r.dropMetric(p) {
-			p.AddTag("hostname", r.hostname)
+			p.AddTag(r.config.HostnameTagName, r.hostname)
 			forward(p)
 		}
 	}
@@ -309,19 +315,19 @@ func (r *metricRouter) Start() {
 
 			case p := <-r.coll_input:
 				coll_forward(p)
-				for i := 0; len(r.coll_input) > 0 && i < r.maxForward; i++ {
+				for i := 0; len(r.coll_input) > 0 && i < (r.maxForward-1); i++ {
 					coll_forward(<-r.coll_input)
 				}
 
 			case p := <-r.recv_input:
 				recv_forward(p)
-				for i := 0; len(r.recv_input) > 0 && i < r.maxForward; i++ {
+				for i := 0; len(r.recv_input) > 0 && i < (r.maxForward-1); i++ {
 					recv_forward(<-r.recv_input)
 				}
 
 			case p := <-r.cache_input:
 				cache_forward(p)
-				for i := 0; len(r.cache_input) > 0 && i < r.maxForward; i++ {
+				for i := 0; len(r.cache_input) > 0 && i < (r.maxForward-1); i++ {
 					cache_forward(<-r.cache_input)
 				}
 			}

receivers/README.md

@@ -6,39 +6,24 @@ This folder contains the ReceiveManager and receiver implementations for the cc-
 
 The configuration file for the receivers is a list of configurations. The `type` field in each specifies which receiver to initialize.
 
-```json
-[
-  {
-    "type": "nats",
-    "address": "nats://my-url",
-    "port" : "4222",
-    "database": "testcluster"
-  }
-]
-```
-
-
-## Type `nats`
-
 ```json
 {
-  "type": "nats",
-  "address": "<nats-URI or hostname>",
-  "port" : "<portnumber>",
-  "database": "<subscribe topic>"
+  "myreceivername" : {
+    "type": "receiver-type",
+    <receiver-specific configuration>
+  }
 }
 ```
 
-The `nats` receiver subscribes to the topic `database` and listens on `address` and `port` for metrics in the InfluxDB line protocol.
+This allows to specify
+
+## Available receivers
+
+- [`nats`](./natsReceiver.md): Receive metrics from the NATS network
+- [`prometheus`](./prometheusReceiver.md): Scrape data from a Prometheus client
+- [`http`](./httpReceiver.md): Listen for HTTP Post requests transporting metrics in InfluxDB line protocol
 
 # Contributing own receivers
-A receiver contains three functions and is derived from the type `Receiver` (in `metricReceiver.go`):
-* `Init(config ReceiverConfig) error`
-* `Start() error`
-* `Close()`
-* `Name() string`
-* `SetSink(sink chan ccMetric.CCMetric)`
+A receiver contains a few functions and is derived from the type `Receiver` (in `metricReceiver.go`):
 
-The data structures should be set up in `Init()` like opening a file or server connection. The `Start()` function should either start a go routine or issue some other asynchronous mechanism for receiving metrics. The `Close()` function should tear down anything created in `Init()`.
-
-Finally, the receiver needs to be registered in the `receiveManager.go`. There is a list of receivers called `AvailableReceivers` which is a map (`receiver_type_string` -> `pointer to Receiver interface`). Add a new entry with a descriptive name and the new receiver.
+For an example, check the [sample receiver](./sampleReceiver.go)

receivers/httpReceiver.go

@@ -0,0 +1,118 @@
+package receivers
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	"github.com/gorilla/mux"
+	influx "github.com/influxdata/line-protocol"
+)
+
+const HTTP_RECEIVER_PORT = "8080"
+
+type HttpReceiverConfig struct {
+	Type string `json:"type"`
+	Addr string `json:"address"`
+	Port string `json:"port"`
+	Path string `json:"path"`
+}
+
+type HttpReceiver struct {
+	receiver
+	handler *influx.MetricHandler
+	parser  *influx.Parser
+	meta    map[string]string
+	config  HttpReceiverConfig
+	router  *mux.Router
+	server  *http.Server
+	wg      sync.WaitGroup
+}
+
+func (r *HttpReceiver) Init(name string, config json.RawMessage) error {
+	r.name = fmt.Sprintf("HttpReceiver(%s)", name)
+	r.config.Port = HTTP_RECEIVER_PORT
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &r.config)
+		if err != nil {
+			cclog.ComponentError(r.name, "Error reading config:", err.Error())
+			return err
+		}
+	}
+	if len(r.config.Port) == 0 {
+		return errors.New("not all configuration variables set required by HttpReceiver")
+	}
+	r.meta = map[string]string{"source": r.name}
+	p := r.config.Path
+	if !strings.HasPrefix(p, "/") {
+		p = "/" + p
+	}
+	uri := fmt.Sprintf("%s:%s%s", r.config.Addr, r.config.Port, p)
+	cclog.ComponentDebug(r.name, "INIT", uri)
+	r.handler = influx.NewMetricHandler()
+	r.parser = influx.NewParser(r.handler)
+	r.parser.SetTimeFunc(DefaultTime)
+
+	r.router = mux.NewRouter()
+	r.router.Path(p).HandlerFunc(r.ServerHttp)
+	r.server = &http.Server{Addr: uri, Handler: r.router}
+	return nil
+}
+
+func (r *HttpReceiver) Start() {
+	cclog.ComponentDebug(r.name, "START")
+	r.wg.Add(1)
+	go func() {
+		err := r.server.ListenAndServe()
+		if err != nil && err.Error() != "http: Server closed" {
+			cclog.ComponentError(r.name, err.Error())
+		}
+		r.wg.Done()
+	}()
+}
+
+func (r *HttpReceiver) ServerHttp(w http.ResponseWriter, req *http.Request) {
+	if req.Method != http.MethodPost {
+		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	body, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	metrics, err := r.parser.Parse(body)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+	}
+
+	for _, m := range metrics {
+		y := lp.FromInfluxMetric(m)
+		for k, v := range r.meta {
+			y.AddMeta(k, v)
+		}
+		if r.sink != nil {
+			r.sink <- y
+		}
+	}
+
+	w.WriteHeader(http.StatusOK)
+}
+
+func (r *HttpReceiver) Close() {
+	r.server.Shutdown(context.Background())
+}
+
+func NewHttpReceiver(name string, config json.RawMessage) (Receiver, error) {
+	r := new(HttpReceiver)
+	err := r.Init(name, config)
+	return r, err
+}

receivers/httpReceiver.md

@@ -0,0 +1,23 @@
+## `http` receiver
+
+The `http` receiver can be used receive metrics through HTTP POST requests.
+
+### Configuration structure
+
+```json
+{
+  "<name>": {
+    "type": "http",
+    "address" : "",
+    "port" : "8080",
+    "path" : "/write"
+  }
+}
+```
+
+- `type`: makes the receiver a `http` receiver
+- `address`: Listen address
+- `port`: Listen port
+- `path`: URL path for the write endpoint
+
+The HTTP endpoint listens to `http://<address>:<port>/<path>`

receivers/metricReceiver.go

@@ -1,9 +1,6 @@
 package receivers
 
 import (
-	//	"time"
-	"encoding/json"
-
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
 )
 
@@ -11,6 +8,7 @@ type defaultReceiverConfig struct {
 	Type string `json:"type"`
 }
 
+// Receiver configuration: Listen address, port
 type ReceiverConfig struct {
 	Addr         string `json:"address"`
 	Port         string `json:"port"`
@@ -20,23 +18,23 @@ type ReceiverConfig struct {
 }
 
 type receiver struct {
-	typename string
-	name     string
-	sink     chan lp.CCMetric
+	name string
+	sink chan lp.CCMetric
 }
 
 type Receiver interface {
-	Init(name string, config json.RawMessage) error
 	Start()
-	Close()
-	Name() string
-	SetSink(sink chan lp.CCMetric)
+	Close()                        // Close / finish metric receiver
+	Name() string                  // Name of the metric receiver
+	SetSink(sink chan lp.CCMetric) // Set sink channel
 }
 
+// Name returns the name of the metric receiver
 func (r *receiver) Name() string {
 	return r.name
 }
 
+// SetSink set the sink channel
 func (r *receiver) SetSink(sink chan lp.CCMetric) {
 	r.sink = sink
 }

receivers/natsReceiver.go

@@ -32,39 +32,6 @@ var DefaultTime = func() time.Time {
 	return time.Unix(42, 0)
 }
 
-func (r *NatsReceiver) Init(name string, config json.RawMessage) error {
-	r.typename = "NatsReceiver"
-	r.name = name
-	r.config.Addr = nats.DefaultURL
-	r.config.Port = "4222"
-	if len(config) > 0 {
-		err := json.Unmarshal(config, &r.config)
-		if err != nil {
-			cclog.ComponentError(r.name, "Error reading config:", err.Error())
-			return err
-		}
-	}
-	if len(r.config.Addr) == 0 ||
-		len(r.config.Port) == 0 ||
-		len(r.config.Subject) == 0 {
-		return errors.New("not all configuration variables set required by NatsReceiver")
-	}
-	r.meta = map[string]string{"source": r.name}
-	uri := fmt.Sprintf("%s:%s", r.config.Addr, r.config.Port)
-	cclog.ComponentDebug(r.name, "INIT", uri, "Subject", r.config.Subject)
-	nc, err := nats.Connect(uri)
-	if err == nil {
-		r.nc = nc
-	} else {
-		r.nc = nil
-		return err
-	}
-	r.handler = influx.NewMetricHandler()
-	r.parser = influx.NewParser(r.handler)
-	r.parser.SetTimeFunc(DefaultTime)
-	return err
-}
-
 func (r *NatsReceiver) Start() {
 	cclog.ComponentDebug(r.name, "START")
 	r.nc.Subscribe(r.config.Subject, r._NatsReceive)
@@ -91,3 +58,35 @@ func (r *NatsReceiver) Close() {
 		r.nc.Close()
 	}
 }
+
+func NewNatsReceiver(name string, config json.RawMessage) (Receiver, error) {
+	r := new(NatsReceiver)
+	r.name = fmt.Sprintf("NatsReceiver(%s)", name)
+	r.config.Addr = nats.DefaultURL
+	r.config.Port = "4222"
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &r.config)
+		if err != nil {
+			cclog.ComponentError(r.name, "Error reading config:", err.Error())
+			return nil, err
+		}
+	}
+	if len(r.config.Addr) == 0 ||
+		len(r.config.Port) == 0 ||
+		len(r.config.Subject) == 0 {
+		return nil, errors.New("not all configuration variables set required by NatsReceiver")
+	}
+	r.meta = map[string]string{"source": r.name}
+	uri := fmt.Sprintf("%s:%s", r.config.Addr, r.config.Port)
+	cclog.ComponentDebug(r.name, "NewNatsReceiver", uri, "Subject", r.config.Subject)
+	if nc, err := nats.Connect(uri); err == nil {
+		r.nc = nc
+	} else {
+		r.nc = nil
+		return nil, err
+	}
+	r.handler = influx.NewMetricHandler()
+	r.parser = influx.NewParser(r.handler)
+	r.parser.SetTimeFunc(DefaultTime)
+	return r, nil
+}

receivers/natsReceiver.md

@@ -0,0 +1,21 @@
+## `nats` receiver
+
+The `nats` receiver can be used receive metrics from the NATS network. The `nats` receiver subscribes to the topic `database` and listens on `address` and `port` for metrics in the InfluxDB line protocol.
+
+### Configuration structure
+
+```json
+{
+  "<name>": {
+    "type": "nats",
+    "address" : "nats-server.example.org",
+    "port" : "4222",
+    "subject" : "subject"
+  }
+}
+```
+
+- `type`: makes the receiver a `nats` receiver
+- `address`: Address of the NATS control server
+- `port`: Port of the NATS control server
+- `subject`: Subscribes to this subject and receive metrics

receivers/prometheusReceiver.go

@@ -0,0 +1,122 @@
+package receivers
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+)
+
+type PrometheusReceiverConfig struct {
+	defaultReceiverConfig
+	Addr     string `json:"address"`
+	Port     string `json:"port"`
+	Path     string `json:"path"`
+	Interval string `json:"interval"`
+	SSL      bool   `json:"ssl"`
+}
+
+type PrometheusReceiver struct {
+	receiver
+	meta     map[string]string
+	config   PrometheusReceiverConfig
+	interval time.Duration
+	done     chan bool
+	wg       sync.WaitGroup
+	ticker   *time.Ticker
+	uri      string
+}
+
+func (r *PrometheusReceiver) Start() {
+	cclog.ComponentDebug(r.name, "START", r.uri)
+	r.wg.Add(1)
+
+	r.ticker = time.NewTicker(r.interval)
+	go func() {
+		for {
+			select {
+			case <-r.done:
+				r.wg.Done()
+				return
+			case t := <-r.ticker.C:
+				resp, err := http.Get(r.uri)
+				if err != nil {
+					log.Fatal(err)
+				}
+				defer resp.Body.Close()
+
+				scanner := bufio.NewScanner(resp.Body)
+				for scanner.Scan() {
+					line := scanner.Text()
+					if strings.HasPrefix(line, "#") {
+						continue
+					}
+					lineSplit := strings.Fields(line)
+					// separate metric name from tags (labels in Prometheus)
+					tags := map[string]string{}
+					name := lineSplit[0]
+					if sindex := strings.Index(name, "{"); sindex >= 0 {
+						eindex := strings.Index(name, "}")
+						for _, kv := range strings.Split(name[sindex+1:eindex], ",") {
+							eq := strings.Index(kv, "=")
+							tags[kv[0:eq]] = strings.Trim(kv[eq+1:], "\"")
+						}
+						name = lineSplit[0][0:sindex]
+					}
+					value, err := strconv.ParseFloat(lineSplit[1], 64)
+					if err == nil {
+						y, err := lp.New(name, tags, r.meta, map[string]interface{}{"value": value}, t)
+						if err == nil {
+							r.sink <- y
+						}
+					}
+				}
+			}
+		}
+	}()
+}
+
+func (r *PrometheusReceiver) Close() {
+	cclog.ComponentDebug(r.name, "CLOSE")
+	r.done <- true
+	r.wg.Wait()
+}
+
+func NewPrometheusReceiver(name string, config json.RawMessage) (Receiver, error) {
+	r := new(PrometheusReceiver)
+	r.name = fmt.Sprintf("PrometheusReceiver(%s)", name)
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &r.config)
+		if err != nil {
+			cclog.ComponentError(r.name, "Error reading config:", err.Error())
+			return nil, err
+		}
+	}
+	if len(r.config.Addr) == 0 ||
+		len(r.config.Port) == 0 ||
+		len(r.config.Interval) == 0 {
+		return nil, errors.New("not all configuration variables set required by PrometheusReceiver (address and port)")
+	}
+	if len(r.config.Interval) > 0 {
+		t, err := time.ParseDuration(r.config.Interval)
+		if err == nil {
+			r.interval = t
+		}
+	}
+	r.meta = map[string]string{"source": r.name}
+	proto := "http"
+	if r.config.SSL {
+		proto = "https"
+	}
+	r.uri = fmt.Sprintf("%s://%s:%s/%s", proto, r.config.Addr, r.config.Port, r.config.Path)
+	return r, nil
+}

receivers/prometheusReceiver.md

@@ -0,0 +1,27 @@
+## `prometheus` receiver
+
+The `prometheus` receiver can be used to scrape the metrics of a single `prometheus` client. It does **not** use any official Golang library but making simple HTTP get requests and parse the response.
+
+### Configuration structure
+
+```json
+{
+  "<name>": {
+    "type": "prometheus",
+    "address" : "testpromhost",
+    "port" : "12345",
+    "path" : "/prometheus",
+    "interval": "5s",
+    "ssl" : true,
+  }
+}
+```
+
+- `type`: makes the receiver a `prometheus` receiver
+- `address`: Hostname or IP of the Prometheus agent
+- `port`: Port of Prometheus agent
+- `path`: Path to the Prometheus endpoint
+- `interval`: Scrape the Prometheus endpoint in this interval (default '5s')
+- `ssl`: Use SSL or not
+
+The receiver requests data from `http(s)://<address>:<port>/<path>`.

receivers/receiveManager.go

@@ -9,8 +9,8 @@ import (
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
 )
 
-var AvailableReceivers = map[string]Receiver{
-	"nats": &NatsReceiver{},
+var AvailableReceivers = map[string]func(name string, config json.RawMessage) (Receiver, error){
+	"nats": NewNatsReceiver,
 }
 
 type receiveManager struct {
@@ -30,11 +30,13 @@ type ReceiveManager interface {
 }
 
 func (rm *receiveManager) Init(wg *sync.WaitGroup, receiverConfigFile string) error {
+	// Initialize struct fields
 	rm.inputs = make([]Receiver, 0)
 	rm.output = nil
 	rm.done = make(chan bool)
 	rm.wg = wg
 	rm.config = make([]json.RawMessage, 0)
+
 	configFile, err := os.Open(receiverConfigFile)
 	if err != nil {
 		cclog.ComponentError("ReceiveManager", err.Error())
@@ -51,6 +53,7 @@ func (rm *receiveManager) Init(wg *sync.WaitGroup, receiverConfigFile string) er
 	for name, raw := range rawConfigs {
 		rm.AddInput(name, raw)
 	}
+
 	return nil
 }
 
@@ -75,10 +78,9 @@ func (rm *receiveManager) AddInput(name string, rawConfig json.RawMessage) error
 		cclog.ComponentError("ReceiveManager", "SKIP", config.Type, "unknown receiver:", err.Error())
 		return err
 	}
-	r := AvailableReceivers[config.Type]
-	err = r.Init(name, rawConfig)
+	r, err := AvailableReceivers[config.Type](name, rawConfig)
 	if err != nil {
-		cclog.ComponentError("ReceiveManager", "SKIP", r.Name(), "initialization failed:", err.Error())
+		cclog.ComponentError("ReceiveManager", "SKIP", name, "initialization failed:", err.Error())
 		return err
 	}
 	rm.inputs = append(rm.inputs, r)

receivers/sampleReceiver.go

@@ -0,0 +1,91 @@
+package receivers
+
+import (
+	"encoding/json"
+	"fmt"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
+)
+
+// SampleReceiver configuration: receiver type, listen address, port
+type SampleReceiverConfig struct {
+	Type string `json:"type"`
+	Addr string `json:"address"`
+	Port string `json:"port"`
+}
+
+type SampleReceiver struct {
+	receiver
+	config SampleReceiverConfig
+
+	// Storage for static information
+	meta map[string]string
+	// Use in case of own go routine
+	// done chan bool
+	// wg   sync.WaitGroup
+}
+
+// Implement functions required for Receiver interface
+// Start(), Close()
+// See: metricReceiver.go
+
+func (r *SampleReceiver) Start() {
+	cclog.ComponentDebug(r.name, "START")
+
+	// Start server process like http.ListenAndServe()
+
+	// or use own go routine but always make sure it exits
+	// as soon as it gets the signal of the r.done channel
+	// r.wg.Add(1)
+	// go func() {
+	// 	for {
+	// 		select {
+	// 		case <-r.done:
+	// 			r.wg.Done()
+	// 			return
+	// 		}
+	// 	}
+	// 	r.wg.Done()
+	// }()
+}
+
+// Close receiver: close network connection, close files, close libraries, ...
+func (r *SampleReceiver) Close() {
+	cclog.ComponentDebug(r.name, "CLOSE")
+
+	// Close server like http.Shutdown()
+
+	// in case of own go routine, send the signal and wait
+	// r.done <- true
+	// r.wg.Wait()
+}
+
+// New function to create a new instance of the receiver
+// Initialize the receiver by giving it a name and reading in the config JSON
+func NewSampleReceiver(name string, config json.RawMessage) (Receiver, error) {
+	r := new(SampleReceiver)
+
+	// Set name of SampleReceiver
+	// The name should be chosen in such a way that different instances of SampleReceiver can be distinguished
+	r.name = fmt.Sprintf("SampleReceiver(%s)", name)
+
+	// Set static information
+	r.meta = map[string]string{"source": r.name}
+
+	// Set defaults in r.config
+	// Allow overwriting these defaults by reading config JSON
+
+	// Read the sample receiver specific JSON config
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &r.config)
+		if err != nil {
+			cclog.ComponentError(r.name, "Error reading config:", err.Error())
+			return nil, err
+		}
+	}
+
+	// Check that all required fields in the configuration are set
+	// Use 'if len(r.config.Option) > 0' for strings
+
+	return r, nil
+}

scripts/cc-metric-collector.deb.control

@@ -0,0 +1,12 @@
+Package: cc-metric-collector
+Version: {VERSION}
+Installed-Size: {INSTALLED_SIZE}
+Architecture: {ARCH}
+Maintainer: thomas.gruber@fau.de
+Depends: libc6 (>= 2.2.1)
+Build-Depends: debhelper-compat (= 13), git, golang-go
+Description: Metric collection daemon from the ClusterCockpit suite
+Homepage: https://github.com/ClusterCockpit/cc-metric-collector
+Source: cc-metric-collector
+Rules-Requires-Root: no
+

scripts/cc-metric-collector.spec

@@ -1,5 +1,5 @@
 Name:           cc-metric-collector
-Version:        0.2
+Version:        %{VERS}
 Release:        1%{?dist}
 Summary:        Metric collection daemon from the ClusterCockpit suite
 
@@ -7,8 +7,9 @@ License:        MIT
 Source0:        %{name}-%{version}.tar.gz
 
 BuildRequires:  go-toolset
-# for internal LIKWID installation
-BuildRequires:  wget perl-Data-Dumper
+BuildRequires:  systemd-rpm-macros
+# for header downloads
+BuildRequires:  wget
 
 Provides:       %{name} = %{version}
 
@@ -34,11 +35,15 @@ install -Dpm 0600 receivers.json %{buildroot}%{_sysconfdir}/%{name}/receivers.js
 install -Dpm 0600 router.json %{buildroot}%{_sysconfdir}/%{name}/router.json
 install -Dpm 0644 scripts/%{name}.service %{buildroot}%{_unitdir}/%{name}.service
 install -Dpm 0600 scripts/%{name}.config %{buildroot}%{_sysconfdir}/default/%{name}
+install -Dpm 0644 scripts/%{name}.sysusers %{buildroot}%{_sysusersdir}/%{name}.conf
 
 
 %check
 # go test should be here... :)
 
+%pre
+%sysusers_create_package scripts/%{name}.sysusers
+
 %post
 %systemd_post %{name}.service
 
@@ -46,17 +51,23 @@ install -Dpm 0600 scripts/%{name}.config %{buildroot}%{_sysconfdir}/default/%{na
 %systemd_preun %{name}.service
 
 %files
+# Binary
+%attr(-,clustercockpit,clustercockpit) %{_sbindir}/%{name}
+# Config
 %dir %{_sysconfdir}/%{name}
-%{_sbindir}/%{name}
+%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/%{name}.json
+%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/collectors.json
+%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/sinks.json
+%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/receivers.json
+%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/router.json
+# Systemd
 %{_unitdir}/%{name}.service
 %{_sysconfdir}/default/%{name}
-%attr(0600,root,root) %config(noreplace) %{_sysconfdir}/%{name}/%{name}.json
-%attr(0600,root,root) %config(noreplace) %{_sysconfdir}/%{name}/collectors.json
-%attr(0600,root,root) %config(noreplace) %{_sysconfdir}/%{name}/sinks.json
-%attr(0600,root,root) %config(noreplace) %{_sysconfdir}/%{name}/receivers.json
-%attr(0600,root,root) %config(noreplace) %{_sysconfdir}/%{name}/router.json
+%{_sysusersdir}/%{name}.conf
 
 %changelog
+* Thu Mar 03 2022 Thomas Gruber - 0.3
+- Add clustercockpit user installation
 * Mon Feb 14 2022 Thomas Gruber - 0.2
 - Add component specific configuration files
 - Add %attr to config files

scripts/cc-metric-collector.sysusers

@@ -0,0 +1,2 @@
+#Type Name           ID GECOS                     Home directory           Shell
+u     clustercockpit -  "User for ClusterCockpit" /run/cc-metric-collector /sbin/nologin

scripts/likwid_perfgroup_to_cc_config.py

@@ -39,7 +39,7 @@ def group_to_json(groupfile):
             llist = re.split("\s+", line)
             calc = llist[-1]
             metric = " ".join(llist[:-1])
-            scope = "hwthread"
+            scope = "cpu"
             if "BOX" in calc:
                 scope = "socket"
             if "PWR" in calc:

sinks/Makefile

@@ -1,12 +0,0 @@
-
-all: libganglia.so
-
-libganglia.so:
-	@find /usr ! -readable -prune -o -type d ! -executable -prune -o -name "$@*" -print0 | \
-	    xargs --null --no-run-if-empty --replace \
-	        ln --symbolic --verbose --force '{}' "$@"
-
-clean:
-	rm -f libganglia.so
-
-.PHONY: clean

sinks/README.md

@@ -6,9 +6,11 @@ This folder contains the SinkManager and sink implementations for the cc-metric-
 - [`stdout`](./stdoutSink.md): Print all metrics to `stdout`, `stderr` or a file
 - [`http`](./httpSink.md): Send metrics to an HTTP server as POST requests
 - [`influxdb`](./influxSink.md): Send metrics to an [InfluxDB](https://www.influxdata.com/products/influxdb/) database
+- [`influxasync`](./influxAsyncSink.md): Send metrics to an [InfluxDB](https://www.influxdata.com/products/influxdb/) database with non-blocking write API
 - [`nats`](./natsSink.md): Publish metrics to the [NATS](https://nats.io/) network overlay system
 - [`ganglia`](./gangliaSink.md): Publish metrics in the [Ganglia Monitoring System](http://ganglia.info/) using the `gmetric` CLI tool
 - [`libganglia`](./libgangliaSink.md): Publish metrics in the [Ganglia Monitoring System](http://ganglia.info/) directly using `libganglia.so`
+- [`prometeus`](./prometheusSink.md): Publish metrics for the [Prometheus Monitoring System](https://prometheus.io/)
 
 # Configuration
 
@@ -34,11 +36,12 @@ The configuration file for the sinks is a list of configurations. The `type` fie
 
 
 # Contributing own sinks
-A sink contains four functions and is derived from the type `sink`:
-* `Init(config json.RawMessage) error`
+A sink contains five functions and is derived from the type `sink`:
+* `Init(name string, config json.RawMessage) error`
 * `Write(point CCMetric) error`
 * `Flush() error`
 * `Close()`
+* `New<Typename>(name string, config json.RawMessage) (Sink, error)` (calls the `Init()` function)
 
 The data structures should be set up in `Init()` like opening a file or server connection. The `Write()` function writes/sends the data. For non-blocking sinks, the `Flush()` method tells the sink to drain its internal buffers. The `Close()` function should tear down anything created in `Init()`.
 
@@ -65,8 +68,8 @@ type SampleSink struct {
 }
 
 // Initialize the sink by giving it a name and reading in the config JSON
-func (s *SampleSink) Init(config json.RawMessage) error {
-	s.name = "SampleSink"   // Always specify a name here
+func (s *SampleSink) Init(name string, config json.RawMessage) error {
+	s.name = fmt.Sprintf("SampleSink(%s)", name)   // Always specify a name here
   // Read in the config JSON
 	if len(config) > 0 {
 		err := json.Unmarshal(config, &s.config)
@@ -91,4 +94,13 @@ func (s *SampleSink) Flush() error {
 
 // Close sink: close network connection, close files, close libraries, ...
 func (s *SampleSink) Close() {}
+
+
+// New function to create a new instance of the sink
+func NewSampleSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(SampleSink)
+	err := s.Init(name, config)
+	return s, err
+}
+
 ```

sinks/gangliaCommon.go

@@ -1,6 +1,7 @@
 package sinks
 
 import (
+	"fmt"
 	"strings"
 
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
@@ -23,11 +24,8 @@ func GangliaMetricName(point lp.CCMetric) string {
 	return name
 }
 
-func GangliaMetricRename(point lp.CCMetric) string {
-	name := point.Name()
-	if name == "mem_total" || name == "swap_total" {
-		return name
-	} else if name == "net_bytes_in" {
+func GangliaMetricRename(name string) string {
+	if name == "net_bytes_in" {
 		return "bytes_in"
 	} else if name == "net_bytes_out" {
 		return "bytes_out"
@@ -48,3 +46,224 @@ func GangliaSlopeType(point lp.CCMetric) uint {
 	}
 	return 3
 }
+
+const DEFAULT_GANGLIA_METRIC_TMAX = 300
+const DEFAULT_GANGLIA_METRIC_SLOPE = "both"
+
+type GangliaMetric struct {
+	Name  string
+	Type  string
+	Slope string
+	Tmax  int
+	Unit  string
+}
+
+type GangliaMetricGroup struct {
+	Name    string
+	Metrics []GangliaMetric
+}
+
+var CommonGangliaMetrics = []GangliaMetricGroup{
+	{
+		Name: "memory",
+		Metrics: []GangliaMetric{
+			{"mem_total", "float", "zero", 1200, "KB"},
+			{"swap_total", "float", "zero", 1200, "KB"},
+			{"mem_free", "float", "both", 180, "KB"},
+			{"mem_shared", "float", "both", 180, "KB"},
+			{"mem_buffers", "float", "both", 180, "KB"},
+			{"mem_cached", "float", "both", 180, "KB"},
+			{"swap_free", "float", "both", 180, "KB"},
+			{"mem_sreclaimable", "float", "both", 180, "KB"},
+			{"mem_slab", "float", "both", 180, "KB"},
+		},
+	},
+	{
+		Name: "cpu",
+		Metrics: []GangliaMetric{
+			{"cpu_num", "uint32", "zero", 1200, "CPUs"},
+			{"cpu_speed", "uint32", "zero", 1200, "MHz"},
+			{"cpu_user", "float", "both", 90, "%"},
+			{"cpu_nice", "float", "both", 90, "%"},
+			{"cpu_system", "float", "both", 90, "%"},
+			{"cpu_idle", "float", "both", 3800, "%"},
+			{"cpu_aidle", "float", "both", 90, "%"},
+			{"cpu_wio", "float", "both", 90, "%"},
+			{"cpu_intr", "float", "both", 90, "%"},
+			{"cpu_sintr", "float", "both", 90, "%"},
+			{"cpu_steal", "float", "both", 90, "%"},
+			{"cpu_guest", "float", "both", 90, "%"},
+			{"cpu_gnice", "float", "both", 90, "%"},
+		},
+	},
+	{
+		Name: "load",
+		Metrics: []GangliaMetric{
+			{"load_one", "float", "both", 70, ""},
+			{"load_five", "float", "both", 325, ""},
+			{"load_fifteen", "float", "both", 950, ""},
+		},
+	},
+	{
+		Name: "disk",
+		Metrics: []GangliaMetric{
+			{"disk_total", "double", "both", 1200, "GB"},
+			{"disk_free", "double", "both", 180, "GB"},
+			{"part_max_used", "float", "both", 180, "%"},
+		},
+	},
+	{
+		Name: "network",
+		Metrics: []GangliaMetric{
+			{"bytes_out", "float", "both", 300, "bytes/sec"},
+			{"bytes_in", "float", "both", 300, "bytes/sec"},
+			{"pkts_in", "float", "both", 300, "packets/sec"},
+			{"pkts_out", "float", "both", 300, "packets/sec"},
+		},
+	},
+	{
+		Name: "process",
+		Metrics: []GangliaMetric{
+			{"proc_run", "uint32", "both", 950, ""},
+			{"proc_total", "uint32", "both", 950, ""},
+		},
+	},
+	{
+		Name: "system",
+		Metrics: []GangliaMetric{
+			{"boottime", "uint32", "zero", 1200, "s"},
+			{"sys_clock", "uint32", "zero", 1200, "s"},
+			{"machine_type", "string", "zero", 1200, ""},
+			{"os_name", "string", "zero", 1200, ""},
+			{"os_release", "string", "zero", 1200, ""},
+			{"mtu", "uint32", "both", 1200, ""},
+		},
+	},
+}
+
+type GangliaMetricConfig struct {
+	Type  string
+	Slope string
+	Tmax  int
+	Unit  string
+	Group string
+	Value string
+	Name  string
+}
+
+func GetCommonGangliaConfig(point lp.CCMetric) GangliaMetricConfig {
+	mname := GangliaMetricRename(point.Name())
+	if oldname, ok := point.GetMeta("oldname"); ok {
+		mname = GangliaMetricRename(oldname)
+	}
+	for _, group := range CommonGangliaMetrics {
+		for _, metric := range group.Metrics {
+			if metric.Name == mname {
+				valueStr := ""
+				value, ok := point.GetField("value")
+				if ok {
+					switch real := value.(type) {
+					case float64:
+						valueStr = fmt.Sprintf("%f", real)
+					case float32:
+						valueStr = fmt.Sprintf("%f", real)
+					case int64:
+						valueStr = fmt.Sprintf("%d", real)
+					case int32:
+						valueStr = fmt.Sprintf("%d", real)
+					case int:
+						valueStr = fmt.Sprintf("%d", real)
+					case uint64:
+						valueStr = fmt.Sprintf("%d", real)
+					case uint32:
+						valueStr = fmt.Sprintf("%d", real)
+					case uint:
+						valueStr = fmt.Sprintf("%d", real)
+					case string:
+						valueStr = real
+					default:
+					}
+				}
+				return GangliaMetricConfig{
+					Group: group.Name,
+					Type:  metric.Type,
+					Slope: metric.Slope,
+					Tmax:  metric.Tmax,
+					Unit:  metric.Unit,
+					Value: valueStr,
+					Name:  GangliaMetricRename(mname),
+				}
+			}
+		}
+	}
+	return GangliaMetricConfig{
+		Group: "",
+		Type:  "",
+		Slope: "",
+		Tmax:  0,
+		Unit:  "",
+		Value: "",
+		Name:  "",
+	}
+}
+
+func GetGangliaConfig(point lp.CCMetric) GangliaMetricConfig {
+	mname := GangliaMetricRename(point.Name())
+	if oldname, ok := point.GetMeta("oldname"); ok {
+		mname = GangliaMetricRename(oldname)
+	}
+	group := ""
+	if g, ok := point.GetMeta("group"); ok {
+		group = g
+	}
+	unit := ""
+	if u, ok := point.GetMeta("unit"); ok {
+		unit = u
+	}
+	valueType := "double"
+	valueStr := ""
+	value, ok := point.GetField("value")
+	if ok {
+		switch real := value.(type) {
+		case float64:
+			valueStr = fmt.Sprintf("%f", real)
+			valueType = "double"
+		case float32:
+			valueStr = fmt.Sprintf("%f", real)
+			valueType = "float"
+		case int64:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "int32"
+		case int32:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "int32"
+		case int:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "int32"
+		case uint64:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "uint32"
+		case uint32:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "uint32"
+		case uint:
+			valueStr = fmt.Sprintf("%d", real)
+			valueType = "uint32"
+		case string:
+			valueStr = real
+			valueType = "string"
+		default:
+			valueType = "invalid"
+		}
+	}
+
+	return GangliaMetricConfig{
+		Group: group,
+		Type:  valueType,
+		Slope: DEFAULT_GANGLIA_METRIC_SLOPE,
+		Tmax:  DEFAULT_GANGLIA_METRIC_TMAX,
+		Unit:  unit,
+		Value: valueStr,
+		Name:  GangliaMetricRename(mname),
+	}
+}

sinks/gangliaSink.go

@@ -24,6 +24,7 @@ type GangliaSinkConfig struct {
 	AddTagsAsDesc   bool   `json:"add_tags_as_desc,omitempty"`
 	ClusterName     string `json:"cluster_name,omitempty"`
 	AddTypeToName   bool   `json:"add_type_to_name,omitempty"`
+	AddUnits        bool   `json:"add_units,omitempty"`
 }
 
 type GangliaSink struct {
@@ -33,16 +34,70 @@ type GangliaSink struct {
 	config         GangliaSinkConfig
 }
 
-func (s *GangliaSink) Init(config json.RawMessage) error {
+func (s *GangliaSink) Write(point lp.CCMetric) error {
 	var err error = nil
-	s.name = "GangliaSink"
+	//var tagsstr []string
+	var argstr []string
+
+	// Get metric config (type, value, ... in suitable format)
+	conf := GetCommonGangliaConfig(point)
+	if len(conf.Type) == 0 {
+		conf = GetGangliaConfig(point)
+	}
+	if len(conf.Type) == 0 {
+		return fmt.Errorf("metric %q (Ganglia name %q) has no 'value' field", point.Name(), conf.Name)
+	}
+
+	if s.config.AddGangliaGroup {
+		argstr = append(argstr, fmt.Sprintf("--group=%s", conf.Group))
+	}
+	if s.config.AddUnits && len(conf.Unit) > 0 {
+		argstr = append(argstr, fmt.Sprintf("--units=%s", conf.Unit))
+	}
+
+	if len(s.config.ClusterName) > 0 {
+		argstr = append(argstr, fmt.Sprintf("--cluster=%s", s.config.ClusterName))
+	}
+	// if s.config.AddTagsAsDesc && len(tagsstr) > 0 {
+	// 	argstr = append(argstr, fmt.Sprintf("--desc=%q", strings.Join(tagsstr, ",")))
+	// }
+	if len(s.gmetric_config) > 0 {
+		argstr = append(argstr, fmt.Sprintf("--conf=%s", s.gmetric_config))
+	}
+	if s.config.AddTypeToName {
+		argstr = append(argstr, fmt.Sprintf("--name=%s", GangliaMetricName(point)))
+	} else {
+		argstr = append(argstr, fmt.Sprintf("--name=%s", conf.Name))
+	}
+	argstr = append(argstr, fmt.Sprintf("--slope=%s", conf.Slope))
+	argstr = append(argstr, fmt.Sprintf("--value=%s", conf.Value))
+	argstr = append(argstr, fmt.Sprintf("--type=%s", conf.Type))
+	argstr = append(argstr, fmt.Sprintf("--tmax=%d", conf.Tmax))
+
+	cclog.ComponentDebug(s.name, s.gmetric_path, strings.Join(argstr, " "))
+	command := exec.Command(s.gmetric_path, argstr...)
+	command.Wait()
+	_, err = command.Output()
+	return err
+}
+
+func (s *GangliaSink) Flush() error {
+	return nil
+}
+
+func (s *GangliaSink) Close() {
+}
+
+func NewGangliaSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(GangliaSink)
+	s.name = fmt.Sprintf("GangliaSink(%s)", name)
 	s.config.AddTagsAsDesc = false
 	s.config.AddGangliaGroup = false
 	if len(config) > 0 {
 		err := json.Unmarshal(config, &s.config)
 		if err != nil {
 			cclog.ComponentError(s.name, "Error reading config for", s.name, ":", err.Error())
-			return err
+			return nil, err
 		}
 	}
 	s.gmetric_path = ""
@@ -60,98 +115,10 @@ func (s *GangliaSink) Init(config json.RawMessage) error {
 		}
 	}
 	if len(s.gmetric_path) == 0 {
-		err = errors.New("cannot find executable 'gmetric'")
+		return nil, errors.New("cannot find executable 'gmetric'")
 	}
 	if len(s.config.GmetricConfig) > 0 {
 		s.gmetric_config = s.config.GmetricConfig
 	}
-	return err
-}
-
-func (s *GangliaSink) Write(point lp.CCMetric) error {
-	var err error = nil
-	var tagsstr []string
-	var argstr []string
-	if s.config.AddGangliaGroup {
-		if point.HasTag("group") {
-			g, _ := point.GetTag("group")
-			argstr = append(argstr, fmt.Sprintf("--group=%s", g))
-		} else if point.HasMeta("group") {
-			g, _ := point.GetMeta("group")
-			argstr = append(argstr, fmt.Sprintf("--group=%s", g))
-		}
-	}
-
-	for key, value := range point.Tags() {
-		switch key {
-		case "unit":
-			argstr = append(argstr, fmt.Sprintf("--units=%s", value))
-		default:
-			tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", key, value))
-		}
-	}
-	if s.config.MetaAsTags {
-		for key, value := range point.Meta() {
-			switch key {
-			case "unit":
-				argstr = append(argstr, fmt.Sprintf("--units=%s", value))
-			default:
-				tagsstr = append(tagsstr, fmt.Sprintf("%s=%s", key, value))
-			}
-		}
-	}
-	if len(s.config.ClusterName) > 0 {
-		argstr = append(argstr, fmt.Sprintf("--cluster=%s", s.config.ClusterName))
-	}
-	if s.config.AddTagsAsDesc && len(tagsstr) > 0 {
-		argstr = append(argstr, fmt.Sprintf("--desc=%q", strings.Join(tagsstr, ",")))
-	}
-	if len(s.gmetric_config) > 0 {
-		argstr = append(argstr, fmt.Sprintf("--conf=%s", s.gmetric_config))
-	}
-	name := GangliaMetricRename(point)
-	if s.config.AddTypeToName {
-		argstr = append(argstr, fmt.Sprintf("--name=%s", GangliaMetricName(point)))
-	} else {
-		argstr = append(argstr, fmt.Sprintf("--name=%s", name))
-	}
-	slope := GangliaSlopeType(point)
-	slopeStr := "both"
-	if slope == 0 {
-		slopeStr = "zero"
-	}
-	argstr = append(argstr, fmt.Sprintf("--slope=%s", slopeStr))
-
-	for k, v := range point.Fields() {
-		if k == "value" {
-			switch value := v.(type) {
-			case float64:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%v", value), "--type=double")
-			case float32:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%v", value), "--type=float")
-			case int:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%d", value), "--type=int32")
-			case int64:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%d", value), "--type=int32")
-			case string:
-				argstr = append(argstr,
-					fmt.Sprintf("--value=%q", value), "--type=string")
-			}
-		}
-	}
-	command := exec.Command(s.gmetric_path, argstr...)
-	command.Wait()
-	_, err = command.Output()
-	return err
-}
-
-func (s *GangliaSink) Flush() error {
-	return nil
-}
-
-func (s *GangliaSink) Close() {
+	return s, nil
 }

sinks/httpSink.go

@@ -38,57 +38,6 @@ type HttpSink struct {
 	flushDelay      time.Duration
 }
 
-func (s *HttpSink) Init(config json.RawMessage) error {
-	// Set default values
-	s.name = "HttpSink"
-	s.config.MaxIdleConns = 10
-	s.config.IdleConnTimeout = "5s"
-	s.config.Timeout = "5s"
-	s.config.FlushDelay = "1s"
-
-	// Read config
-	if len(config) > 0 {
-		err := json.Unmarshal(config, &s.config)
-		if err != nil {
-			return err
-		}
-	}
-	if len(s.config.URL) == 0 {
-		return errors.New("`url` config option is required for HTTP sink")
-	}
-	if s.config.MaxIdleConns > 0 {
-		s.maxIdleConns = s.config.MaxIdleConns
-	}
-	if len(s.config.IdleConnTimeout) > 0 {
-		t, err := time.ParseDuration(s.config.IdleConnTimeout)
-		if err == nil {
-			s.idleConnTimeout = t
-		}
-	}
-	if len(s.config.Timeout) > 0 {
-		t, err := time.ParseDuration(s.config.Timeout)
-		if err == nil {
-			s.timeout = t
-		}
-	}
-	if len(s.config.FlushDelay) > 0 {
-		t, err := time.ParseDuration(s.config.FlushDelay)
-		if err == nil {
-			s.flushDelay = t
-		}
-	}
-	tr := &http.Transport{
-		MaxIdleConns:    s.maxIdleConns,
-		IdleConnTimeout: s.idleConnTimeout,
-	}
-	s.client = &http.Client{Transport: tr, Timeout: s.timeout}
-	s.buffer = &bytes.Buffer{}
-	s.encoder = influx.NewEncoder(s.buffer)
-	s.encoder.SetPrecision(time.Second)
-
-	return nil
-}
-
 func (s *HttpSink) Write(m lp.CCMetric) error {
 	if s.buffer.Len() == 0 && s.flushDelay != 0 {
 		// This is the first write since the last flush, start the flushTimer!
@@ -104,7 +53,7 @@ func (s *HttpSink) Write(m lp.CCMetric) error {
 		})
 	}
 
-	p := m.ToPoint(s.config.MetaAsTags)
+	p := m.ToPoint(s.meta_as_tags)
 
 	s.lock.Lock()
 	_, err := s.encoder.Encode(p)
@@ -169,3 +118,59 @@ func (s *HttpSink) Close() {
 	}
 	s.client.CloseIdleConnections()
 }
+
+func NewHttpSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(HttpSink)
+	// Set default values
+	s.name = fmt.Sprintf("HttpSink(%s)", name)
+	s.config.MaxIdleConns = 10
+	s.config.IdleConnTimeout = "5s"
+	s.config.Timeout = "5s"
+	s.config.FlushDelay = "1s"
+
+	// Read config
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &s.config)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if len(s.config.URL) == 0 {
+		return nil, errors.New("`url` config option is required for HTTP sink")
+	}
+	if s.config.MaxIdleConns > 0 {
+		s.maxIdleConns = s.config.MaxIdleConns
+	}
+	if len(s.config.IdleConnTimeout) > 0 {
+		t, err := time.ParseDuration(s.config.IdleConnTimeout)
+		if err == nil {
+			s.idleConnTimeout = t
+		}
+	}
+	if len(s.config.Timeout) > 0 {
+		t, err := time.ParseDuration(s.config.Timeout)
+		if err == nil {
+			s.timeout = t
+		}
+	}
+	if len(s.config.FlushDelay) > 0 {
+		t, err := time.ParseDuration(s.config.FlushDelay)
+		if err == nil {
+			s.flushDelay = t
+		}
+	}
+	// Create lookup map to use meta infos as tags in the output metric
+	s.meta_as_tags = make(map[string]bool)
+	for _, k := range s.config.MetaAsTags {
+		s.meta_as_tags[k] = true
+	}
+	tr := &http.Transport{
+		MaxIdleConns:    s.maxIdleConns,
+		IdleConnTimeout: s.idleConnTimeout,
+	}
+	s.client = &http.Client{Transport: tr, Timeout: s.timeout}
+	s.buffer = &bytes.Buffer{}
+	s.encoder = influx.NewEncoder(s.buffer)
+	s.encoder.SetPrecision(time.Second)
+	return s, nil
+}

sinks/influxAsyncSink.go

@@ -1,10 +1,12 @@
 package sinks
 
 import (
+	"context"
 	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"time"
 
 	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
@@ -25,16 +27,21 @@ type InfluxAsyncSinkConfig struct {
 	// Maximum number of points sent to server in single request. Default 5000
 	BatchSize uint `json:"batch_size,omitempty"`
 	// Interval, in ms, in which is buffer flushed if it has not been already written (by reaching batch size) . Default 1000ms
-	FlushInterval uint `json:"flush_interval,omitempty"`
+	FlushInterval         uint   `json:"flush_interval,omitempty"`
+	InfluxRetryInterval   string `json:"retry_interval"`
+	InfluxExponentialBase uint   `json:"retry_exponential_base"`
+	InfluxMaxRetries      uint   `json:"max_retries"`
+	InfluxMaxRetryTime    string `json:"max_retry_time"`
 }
 
 type InfluxAsyncSink struct {
 	sink
-	client    influxdb2.Client
-	writeApi  influxdb2Api.WriteAPI
-	retPolicy string
-	errors    <-chan error
-	config    InfluxAsyncSinkConfig
+	client              influxdb2.Client
+	writeApi            influxdb2Api.WriteAPI
+	errors              <-chan error
+	config              InfluxAsyncSinkConfig
+	influxRetryInterval uint
+	influxMaxRetryTime  uint
 }
 
 func (s *InfluxAsyncSink) connect() error {
@@ -63,47 +70,26 @@ func (s *InfluxAsyncSink) connect() error {
 			InsecureSkipVerify: true,
 		},
 	)
+	clientOptions.SetMaxRetryInterval(s.influxRetryInterval)
+	clientOptions.SetMaxRetryTime(s.influxMaxRetryTime)
+	clientOptions.SetExponentialBase(s.config.InfluxExponentialBase)
+	clientOptions.SetMaxRetries(s.config.InfluxMaxRetries)
+
 	s.client = influxdb2.NewClientWithOptions(uri, auth, clientOptions)
 	s.writeApi = s.client.WriteAPI(s.config.Organization, s.config.Database)
+	ok, err := s.client.Ping(context.Background())
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return fmt.Errorf("connection to %s not healthy", uri)
+	}
 	return nil
 }
 
-func (s *InfluxAsyncSink) Init(config json.RawMessage) error {
-	s.name = "InfluxSink"
-
-	// Set default for maximum number of points sent to server in single request.
-	s.config.BatchSize = 100
-
-	if len(config) > 0 {
-		err := json.Unmarshal(config, &s.config)
-		if err != nil {
-			return err
-		}
-	}
-	if len(s.config.Host) == 0 ||
-		len(s.config.Port) == 0 ||
-		len(s.config.Database) == 0 ||
-		len(s.config.Organization) == 0 ||
-		len(s.config.Password) == 0 {
-		return errors.New("not all configuration variables set required by InfluxAsyncSink")
-	}
-
-	// Connect to InfluxDB server
-	err := s.connect()
-
-	// Start background: Read from error channel
-	s.errors = s.writeApi.Errors()
-	go func() {
-		for err := range s.errors {
-			cclog.ComponentError(s.name, err.Error())
-		}
-	}()
-	return err
-}
-
 func (s *InfluxAsyncSink) Write(m lp.CCMetric) error {
 	s.writeApi.WritePoint(
-		m.ToPoint(s.config.MetaAsTags),
+		m.ToPoint(s.meta_as_tags),
 	)
 	return nil
 }
@@ -118,3 +104,82 @@ func (s *InfluxAsyncSink) Close() {
 	s.writeApi.Flush()
 	s.client.Close()
 }
+
+func NewInfluxAsyncSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(InfluxAsyncSink)
+	s.name = fmt.Sprintf("InfluxSink(%s)", name)
+
+	// Set default for maximum number of points sent to server in single request.
+	s.config.BatchSize = 100
+	s.influxRetryInterval = uint(time.Duration(1) * time.Second)
+	s.config.InfluxRetryInterval = "1s"
+	s.influxMaxRetryTime = uint(7 * time.Duration(24) * time.Hour)
+	s.config.InfluxMaxRetryTime = "168h"
+	s.config.InfluxMaxRetries = 20
+	s.config.InfluxExponentialBase = 2
+
+	// Default retry intervals (in seconds)
+	// 1 2
+	// 2 4
+	// 4 8
+	// 8 16
+	// 16 32
+	// 32 64
+	// 64 128
+	// 128 256
+	// 256 512
+	// 512 1024
+	// 1024 2048
+	// 2048 4096
+	// 4096 8192
+	// 8192 16384
+	// 16384 32768
+	// 32768 65536
+	// 65536 131072
+	// 131072 262144
+	// 262144 524288
+
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &s.config)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if len(s.config.Host) == 0 ||
+		len(s.config.Port) == 0 ||
+		len(s.config.Database) == 0 ||
+		len(s.config.Organization) == 0 ||
+		len(s.config.Password) == 0 {
+		return nil, errors.New("not all configuration variables set required by InfluxAsyncSink")
+	}
+	// Create lookup map to use meta infos as tags in the output metric
+	s.meta_as_tags = make(map[string]bool)
+	for _, k := range s.config.MetaAsTags {
+		s.meta_as_tags[k] = true
+	}
+
+	toUint := func(duration string, def uint) uint {
+		t, err := time.ParseDuration(duration)
+		if err == nil {
+			return uint(t.Milliseconds())
+		}
+		return def
+	}
+	s.influxRetryInterval = toUint(s.config.InfluxRetryInterval, s.influxRetryInterval)
+	s.influxMaxRetryTime = toUint(s.config.InfluxMaxRetryTime, s.influxMaxRetryTime)
+
+	// Connect to InfluxDB server
+	if err := s.connect(); err != nil {
+		return nil, fmt.Errorf("unable to connect: %v", err)
+	}
+
+	// Start background: Read from error channel
+	s.errors = s.writeApi.Errors()
+	go func() {
+		for err := range s.errors {
+			cclog.ComponentError(s.name, err.Error())
+		}
+	}()
+
+	return s, nil
+}

sinks/influxAsyncSink.md

@@ -18,6 +18,10 @@ The `influxasync` sink uses the official [InfluxDB golang client](https://pkg.go
     "organization": "myorg",
     "ssl": true,
     "batch_size": 200,
+    "retry_interval" : "1s",
+    "retry_exponential_base" : 2,
+    "max_retries": 20,
+    "max_retry_time" : "168h"
   }
 }
 ```
@@ -31,4 +35,10 @@ The `influxasync` sink uses the official [InfluxDB golang client](https://pkg.go
 - `password`: Password for basic authentification
 - `organization`: Organization in the InfluxDB
 - `ssl`: Use SSL connection
-- `batch_size`: batch up metrics internally, default 100
+- `batch_size`: batch up metrics internally, default 100
+- `retry_interval`: Base retry interval for failed write requests, default 1s
+- `retry_exponential_base`: The retry interval is exponentially increased with this base, default 2
+- `max_retries`: Maximal number of retry attempts
+- `max_retry_time`: Maximal time to retry failed writes, default 168h (one week)
+
+For information about the calculation of the retry interval settings, see [offical influxdb-client-go documentation](https://github.com/influxdata/influxdb-client-go#handling-of-failed-async-writes)

sinks/influxSink.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"time"
 
 	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
@@ -15,21 +16,29 @@ import (
 
 type InfluxSinkConfig struct {
 	defaultSinkConfig
-	Host         string `json:"host,omitempty"`
-	Port         string `json:"port,omitempty"`
-	Database     string `json:"database,omitempty"`
-	User         string `json:"user,omitempty"`
-	Password     string `json:"password,omitempty"`
-	Organization string `json:"organization,omitempty"`
-	SSL          bool   `json:"ssl,omitempty"`
-	RetentionPol string `json:"retention_policy,omitempty"`
+	Host                  string `json:"host,omitempty"`
+	Port                  string `json:"port,omitempty"`
+	Database              string `json:"database,omitempty"`
+	User                  string `json:"user,omitempty"`
+	Password              string `json:"password,omitempty"`
+	Organization          string `json:"organization,omitempty"`
+	SSL                   bool   `json:"ssl,omitempty"`
+	RetentionPol          string `json:"retention_policy,omitempty"`
+	InfluxRetryInterval   string `json:"retry_interval"`
+	InfluxExponentialBase uint   `json:"retry_exponential_base"`
+	InfluxMaxRetries      uint   `json:"max_retries"`
+	InfluxMaxRetryTime    string `json:"max_retry_time"`
+	//InfluxMaxRetryDelay  string `json:"max_retry_delay"` // It is mentioned in the docs but there is no way to set it
 }
 
 type InfluxSink struct {
 	sink
-	client   influxdb2.Client
-	writeApi influxdb2Api.WriteAPIBlocking
-	config   InfluxSinkConfig
+	client              influxdb2.Client
+	writeApi            influxdb2Api.WriteAPIBlocking
+	config              InfluxSinkConfig
+	influxRetryInterval uint
+	influxMaxRetryTime  uint
+	//influxMaxRetryDelay uint
 }
 
 func (s *InfluxSink) connect() error {
@@ -52,36 +61,29 @@ func (s *InfluxSink) connect() error {
 			InsecureSkipVerify: true,
 		},
 	)
+
+	clientOptions.SetMaxRetryInterval(s.influxRetryInterval)
+	clientOptions.SetMaxRetryTime(s.influxMaxRetryTime)
+	clientOptions.SetExponentialBase(s.config.InfluxExponentialBase)
+	clientOptions.SetMaxRetries(s.config.InfluxMaxRetries)
+
 	s.client = influxdb2.NewClientWithOptions(uri, auth, clientOptions)
 	s.writeApi = s.client.WriteAPIBlocking(s.config.Organization, s.config.Database)
+	ok, err := s.client.Ping(context.Background())
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return fmt.Errorf("connection to %s not healthy", uri)
+	}
 	return nil
 }
 
-func (s *InfluxSink) Init(config json.RawMessage) error {
-	s.name = "InfluxSink"
-	if len(config) > 0 {
-		err := json.Unmarshal(config, &s.config)
-		if err != nil {
-			return err
-		}
-	}
-	if len(s.config.Host) == 0 ||
-		len(s.config.Port) == 0 ||
-		len(s.config.Database) == 0 ||
-		len(s.config.Organization) == 0 ||
-		len(s.config.Password) == 0 {
-		return errors.New("not all configuration variables set required by InfluxSink")
-	}
-
-	// Connect to InfluxDB server
-	return s.connect()
-}
-
 func (s *InfluxSink) Write(m lp.CCMetric) error {
 	err :=
 		s.writeApi.WritePoint(
 			context.Background(),
-			m.ToPoint(s.config.MetaAsTags),
+			m.ToPoint(s.meta_as_tags),
 		)
 	return err
 }
@@ -94,3 +96,49 @@ func (s *InfluxSink) Close() {
 	cclog.ComponentDebug(s.name, "Closing InfluxDB connection")
 	s.client.Close()
 }
+
+func NewInfluxSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(InfluxSink)
+	s.name = fmt.Sprintf("InfluxSink(%s)", name)
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &s.config)
+		if err != nil {
+			return nil, err
+		}
+	}
+	s.influxRetryInterval = uint(time.Duration(1) * time.Second)
+	s.config.InfluxRetryInterval = "1s"
+	s.influxMaxRetryTime = uint(7 * time.Duration(24) * time.Hour)
+	s.config.InfluxMaxRetryTime = "168h"
+	s.config.InfluxMaxRetries = 20
+	s.config.InfluxExponentialBase = 2
+
+	if len(s.config.Host) == 0 ||
+		len(s.config.Port) == 0 ||
+		len(s.config.Database) == 0 ||
+		len(s.config.Organization) == 0 ||
+		len(s.config.Password) == 0 {
+		return nil, errors.New("not all configuration variables set required by InfluxSink")
+	}
+	// Create lookup map to use meta infos as tags in the output metric
+	s.meta_as_tags = make(map[string]bool)
+	for _, k := range s.config.MetaAsTags {
+		s.meta_as_tags[k] = true
+	}
+
+	toUint := func(duration string, def uint) uint {
+		t, err := time.ParseDuration(duration)
+		if err == nil {
+			return uint(t.Milliseconds())
+		}
+		return def
+	}
+	s.influxRetryInterval = toUint(s.config.InfluxRetryInterval, s.influxRetryInterval)
+	s.influxMaxRetryTime = toUint(s.config.InfluxMaxRetryTime, s.influxMaxRetryTime)
+
+	// Connect to InfluxDB server
+	if err := s.connect(); err != nil {
+		return nil, fmt.Errorf("unable to connect: %v", err)
+	}
+	return s, nil
+}

sinks/influxSink.md

@@ -17,6 +17,10 @@ The `influxdb` sink uses the official [InfluxDB golang client](https://pkg.go.de
     "password" : "examplepw",
     "organization": "myorg",
     "ssl": true,
+    "retry_interval" : "1s",
+    "retry_exponential_base" : 2,
+    "max_retries": 20,
+    "max_retry_time" : "168h"
   }
 }
 ```
@@ -29,4 +33,10 @@ The `influxdb` sink uses the official [InfluxDB golang client](https://pkg.go.de
 - `user`: Username for basic authentification
 - `password`: Password for basic authentification
 - `organization`: Organization in the InfluxDB
-- `ssl`: Use SSL connection
+- `ssl`: Use SSL connection
+- `retry_interval`: Base retry interval for failed write requests, default 1s
+- `retry_exponential_base`: The retry interval is exponentially increased with this base, default 2
+- `max_retries`: Maximal number of retry attempts
+- `max_retry_time`: Maximal time to retry failed writes, default 168h (one week)
+
+For information about the calculation of the retry interval settings, see [offical influxdb-client-go documentation](https://github.com/influxdata/influxdb-client-go#handling-of-failed-async-writes)

sinks/libgangliaSink.go

@@ -82,21 +82,21 @@ const (
 	GMOND_CONFIG_FILE    = `/etc/ganglia/gmond.conf`
 )
 
-type LibgangliaSinkSpecialMetric struct {
-	MetricName string `json:"metric_name,omitempty"`
-	NewName    string `json:"new_name,omitempty"`
-	Slope      string `json:"slope,omitempty"`
-}
+// type LibgangliaSinkSpecialMetric struct {
+// 	MetricName string `json:"metric_name,omitempty"`
+// 	NewName    string `json:"new_name,omitempty"`
+// 	Slope      string `json:"slope,omitempty"`
+// }
 
 type LibgangliaSinkConfig struct {
 	defaultSinkConfig
-	GangliaLib      string                                 `json:"libganglia_path,omitempty"`
-	GmondConfig     string                                 `json:"gmond_config,omitempty"`
-	AddGangliaGroup bool                                   `json:"add_ganglia_group,omitempty"`
-	AddTypeToName   bool                                   `json:"add_type_to_name,omitempty"`
-	AddUnits        bool                                   `json:"add_units,omitempty"`
-	ClusterName     string                                 `json:"cluster_name,omitempty"`
-	SpecialMetrics  map[string]LibgangliaSinkSpecialMetric `json:"rename_metrics,omitempty"` // Map to rename metric name from key to value
+	GangliaLib      string `json:"libganglia_path,omitempty"`
+	GmondConfig     string `json:"gmond_config,omitempty"`
+	AddGangliaGroup bool   `json:"add_ganglia_group,omitempty"`
+	AddTypeToName   bool   `json:"add_type_to_name,omitempty"`
+	AddUnits        bool   `json:"add_units,omitempty"`
+	ClusterName     string `json:"cluster_name,omitempty"`
+	//SpecialMetrics  map[string]LibgangliaSinkSpecialMetric `json:"rename_metrics,omitempty"` // Map to rename metric name from key to value
 	//AddTagsAsDesc   bool              `json:"add_tags_as_desc,omitempty"`
 }
 
@@ -109,65 +109,6 @@ type LibgangliaSink struct {
 	cstrCache      map[string]*C.char
 }
 
-func (s *LibgangliaSink) Init(config json.RawMessage) error {
-	var err error = nil
-	s.name = "LibgangliaSink"
-	//s.config.AddTagsAsDesc = false
-	s.config.AddGangliaGroup = false
-	s.config.AddTypeToName = false
-	s.config.AddUnits = true
-	s.config.GmondConfig = string(GMOND_CONFIG_FILE)
-	s.config.GangliaLib = string(GANGLIA_LIB_NAME)
-	if len(config) > 0 {
-		err = json.Unmarshal(config, &s.config)
-		if err != nil {
-			cclog.ComponentError(s.name, "Error reading config:", err.Error())
-			return err
-		}
-	}
-	lib := dl.New(s.config.GangliaLib, GANGLIA_LIB_DL_FLAGS)
-	if lib == nil {
-		return fmt.Errorf("error instantiating DynamicLibrary for %s", s.config.GangliaLib)
-	}
-	err = lib.Open()
-	if err != nil {
-		return fmt.Errorf("error opening %s: %v", s.config.GangliaLib, err)
-	}
-
-	// Set up cache for the C strings
-	s.cstrCache = make(map[string]*C.char)
-	// s.cstrCache["globals"] = C.CString("globals")
-
-	// s.cstrCache["override_hostname"] = C.CString("override_hostname")
-	// s.cstrCache["override_ip"] = C.CString("override_ip")
-
-	// Add some constant strings
-	s.cstrCache["GROUP"] = C.CString("GROUP")
-	s.cstrCache["CLUSTER"] = C.CString("CLUSTER")
-	s.cstrCache[""] = C.CString("")
-
-	// Add cluster name for lookup in Write()
-	if len(s.config.ClusterName) > 0 {
-		s.cstrCache[s.config.ClusterName] = C.CString(s.config.ClusterName)
-	}
-	// Add supported types for later lookup in Write()
-	s.cstrCache["double"] = C.CString("double")
-	s.cstrCache["int32"] = C.CString("int32")
-	s.cstrCache["string"] = C.CString("string")
-
-	// Create Ganglia pool
-	s.global_context = C.Ganglia_pool_create(nil)
-	// Load Ganglia configuration
-	s.cstrCache[s.config.GmondConfig] = C.CString(s.config.GmondConfig)
-	s.gmond_config = C.Ganglia_gmond_config_create(s.cstrCache[s.config.GmondConfig], 0)
-	//globals := C.cfg_getsec(gmond_config, s.cstrCache["globals"])
-	//override_hostname := C.cfg_getstr(globals, s.cstrCache["override_hostname"])
-	//override_ip := C.cfg_getstr(globals, s.cstrCache["override_ip"])
-
-	s.send_channels = C.Ganglia_udp_send_channels_create(s.global_context, s.gmond_config)
-	return nil
-}
-
 func (s *LibgangliaSink) Write(point lp.CCMetric) error {
 	var err error = nil
 	var c_name *C.char
@@ -183,73 +124,46 @@ func (s *LibgangliaSink) Write(point lp.CCMetric) error {
 		return s.cstrCache[key]
 	}
 
-	// Get metric name
-	metricname := GangliaMetricRename(point)
-	if s.config.AddTypeToName {
-		c_name = lookup(GangliaMetricName(point))
-	} else {
-		c_name = lookup(metricname)
+	conf := GetCommonGangliaConfig(point)
+	if len(conf.Type) == 0 {
+		conf = GetGangliaConfig(point)
+	}
+	if len(conf.Type) == 0 {
+		return fmt.Errorf("metric %q (Ganglia name %q) has no 'value' field", point.Name(), conf.Name)
 	}
 
-	// Get the value C string and lookup the type string in the cache
-	value, ok := point.GetField("value")
-	if !ok {
-		return fmt.Errorf("metric %s has no 'value' field", metricname)
-	}
-	switch real := value.(type) {
-	case float64:
-		c_value = C.CString(fmt.Sprintf("%f", real))
-		c_type = lookup("double")
-	case float32:
-		c_value = C.CString(fmt.Sprintf("%f", real))
-		c_type = lookup("float")
-	case int64:
-		c_value = C.CString(fmt.Sprintf("%d", real))
-		c_type = lookup("int32")
-	case int32:
-		c_value = C.CString(fmt.Sprintf("%d", real))
-		c_type = lookup("int32")
-	case int:
-		c_value = C.CString(fmt.Sprintf("%d", real))
-		c_type = lookup("int32")
-	case string:
-		c_value = C.CString(real)
-		c_type = lookup("string")
-	default:
-		return fmt.Errorf("metric %s has invalid 'value' type for %s", point.Name(), s.name)
+	if s.config.AddTypeToName {
+		conf.Name = GangliaMetricName(point)
 	}
 
+	c_value = C.CString(conf.Value)
+	c_type = lookup(conf.Type)
+	c_name = lookup(conf.Name)
+
 	// Add unit
+	unit := ""
 	if s.config.AddUnits {
-		if tagunit, tagok := point.GetTag("unit"); tagok {
-			c_unit = lookup(tagunit)
-		} else if metaunit, metaok := point.GetMeta("unit"); metaok {
-			c_unit = lookup(metaunit)
-		} else {
-			c_unit = lookup("")
-		}
-	} else {
-		c_unit = lookup("")
+		unit = conf.Unit
 	}
+	c_unit = lookup(unit)
 
 	// Determine the slope of the metric. Ganglia's own collector mostly use
 	// 'both' but the mem and swap total uses 'zero'.
-	slope := GangliaSlopeType(point)
 	slope_type := C.GANGLIA_SLOPE_BOTH
-	switch slope {
-	case 0:
+	switch conf.Slope {
+	case "zero":
 		slope_type = C.GANGLIA_SLOPE_ZERO
+	case "both":
+		slope_type = C.GANGLIA_SLOPE_BOTH
 	}
 
 	// Create a new Ganglia metric
 	gmetric := C.Ganglia_metric_create(s.global_context)
 	// Set name, value, type and unit in the Ganglia metric
-	// Since we don't have this information from the collectors,
-	// we assume that the metric value can go up and down (slope),
-	// and there is no maximum for 'dmax' and 'tmax'.
-	// Ganglia's collectors set 'tmax' but not 'dmax'
+	// The default slope_type is both directions, so up and down. Some metrics want 'zero' slope, probably constant.
+	// The 'tmax' value is by default 300.
 	rval := C.int(0)
-	rval = C.Ganglia_metric_set(gmetric, c_name, c_value, c_type, c_unit, C.uint(slope_type), 0, 0)
+	rval = C.Ganglia_metric_set(gmetric, c_name, c_value, c_type, c_unit, C.uint(slope_type), C.uint(conf.Tmax), 0)
 	switch rval {
 	case 1:
 		C.free(unsafe.Pointer(c_value))
@@ -259,10 +173,10 @@ func (s *LibgangliaSink) Write(point lp.CCMetric) error {
 		return errors.New("one of your parameters has an invalid character '\"'")
 	case 3:
 		C.free(unsafe.Pointer(c_value))
-		return fmt.Errorf("the type parameter \"%s\" is not a valid type", C.GoString(c_type))
+		return fmt.Errorf("the type parameter \"%s\" is not a valid type", conf.Type)
 	case 4:
 		C.free(unsafe.Pointer(c_value))
-		return fmt.Errorf("the value parameter \"%s\" does not represent a number", C.GoString(c_value))
+		return fmt.Errorf("the value parameter \"%s\" does not represent a number", conf.Value)
 	default:
 	}
 
@@ -271,8 +185,8 @@ func (s *LibgangliaSink) Write(point lp.CCMetric) error {
 		C.Ganglia_metadata_add(gmetric, lookup("CLUSTER"), lookup(s.config.ClusterName))
 	}
 	// Set the group metadata in the Ganglia metric if configured
-	if group, ok := point.GetMeta("group"); ok && s.config.AddGangliaGroup {
-		c_group := lookup(group)
+	if s.config.AddGangliaGroup {
+		c_group := lookup(conf.Group)
 		C.Ganglia_metadata_add(gmetric, lookup("GROUP"), c_group)
 	}
 
@@ -307,3 +221,63 @@ func (s *LibgangliaSink) Close() {
 		C.free(unsafe.Pointer(cstr))
 	}
 }
+
+func NewLibgangliaSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(LibgangliaSink)
+	var err error = nil
+	s.name = fmt.Sprintf("LibgangliaSink(%s)", name)
+	//s.config.AddTagsAsDesc = false
+	s.config.AddGangliaGroup = false
+	s.config.AddTypeToName = false
+	s.config.AddUnits = true
+	s.config.GmondConfig = string(GMOND_CONFIG_FILE)
+	s.config.GangliaLib = string(GANGLIA_LIB_NAME)
+	if len(config) > 0 {
+		err = json.Unmarshal(config, &s.config)
+		if err != nil {
+			cclog.ComponentError(s.name, "Error reading config:", err.Error())
+			return nil, err
+		}
+	}
+	lib := dl.New(s.config.GangliaLib, GANGLIA_LIB_DL_FLAGS)
+	if lib == nil {
+		return nil, fmt.Errorf("error instantiating DynamicLibrary for %s", s.config.GangliaLib)
+	}
+	err = lib.Open()
+	if err != nil {
+		return nil, fmt.Errorf("error opening %s: %v", s.config.GangliaLib, err)
+	}
+
+	// Set up cache for the C strings
+	s.cstrCache = make(map[string]*C.char)
+	// s.cstrCache["globals"] = C.CString("globals")
+
+	// s.cstrCache["override_hostname"] = C.CString("override_hostname")
+	// s.cstrCache["override_ip"] = C.CString("override_ip")
+
+	// Add some constant strings
+	s.cstrCache["GROUP"] = C.CString("GROUP")
+	s.cstrCache["CLUSTER"] = C.CString("CLUSTER")
+	s.cstrCache[""] = C.CString("")
+
+	// Add cluster name for lookup in Write()
+	if len(s.config.ClusterName) > 0 {
+		s.cstrCache[s.config.ClusterName] = C.CString(s.config.ClusterName)
+	}
+	// Add supported types for later lookup in Write()
+	s.cstrCache["double"] = C.CString("double")
+	s.cstrCache["int32"] = C.CString("int32")
+	s.cstrCache["string"] = C.CString("string")
+
+	// Create Ganglia pool
+	s.global_context = C.Ganglia_pool_create(nil)
+	// Load Ganglia configuration
+	s.cstrCache[s.config.GmondConfig] = C.CString(s.config.GmondConfig)
+	s.gmond_config = C.Ganglia_gmond_config_create(s.cstrCache[s.config.GmondConfig], 0)
+	//globals := C.cfg_getsec(gmond_config, s.cstrCache["globals"])
+	//override_hostname := C.cfg_getstr(globals, s.cstrCache["override_hostname"])
+	//override_ip := C.cfg_getstr(globals, s.cstrCache["override_ip"])
+
+	s.send_channels = C.Ganglia_udp_send_channels_create(s.global_context, s.gmond_config)
+	return s, nil
+}

sinks/metricSink.go

@@ -1,29 +1,27 @@
 package sinks
 
 import (
-	"encoding/json"
-
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
 )
 
 type defaultSinkConfig struct {
-	MetaAsTags bool   `json:"meta_as_tags,omitempty"`
-	Type       string `json:"type"`
+	MetaAsTags []string `json:"meta_as_tags,omitempty"`
+	Type       string   `json:"type"`
 }
 
 type sink struct {
-	meta_as_tags bool
-	name         string
+	meta_as_tags map[string]bool // Use meta data tags as tags
+	name         string          // Name of the sink
 }
 
 type Sink interface {
-	Init(config json.RawMessage) error
-	Write(point lp.CCMetric) error
-	Flush() error
-	Close()
-	Name() string
+	Write(point lp.CCMetric) error // Write metric to the sink
+	Flush() error                  // Flush buffered metrics
+	Close()                        // Close / finish metric sink
+	Name() string                  // Name of the metric sink
 }
 
+// Name returns the name of the metric sink
 func (s *sink) Name() string {
 	return s.name
 }

sinks/natsSink.go

@@ -53,33 +53,9 @@ func (s *NatsSink) connect() error {
 	return nil
 }
 
-func (s *NatsSink) Init(config json.RawMessage) error {
-	s.name = "NatsSink"
-	if len(config) > 0 {
-		err := json.Unmarshal(config, &s.config)
-		if err != nil {
-			cclog.ComponentError(s.name, "Error reading config for", s.name, ":", err.Error())
-			return err
-		}
-	}
-	if len(s.config.Host) == 0 ||
-		len(s.config.Port) == 0 ||
-		len(s.config.Database) == 0 {
-		return errors.New("not all configuration variables set required by NatsSink")
-	}
-	// Setup Influx line protocol
-	s.buffer = &bytes.Buffer{}
-	s.buffer.Grow(1025)
-	s.encoder = influx.NewEncoder(s.buffer)
-	s.encoder.SetPrecision(time.Second)
-	s.encoder.SetMaxLineBytes(1024)
-	// Setup infos for connection
-	return s.connect()
-}
-
 func (s *NatsSink) Write(m lp.CCMetric) error {
 	if s.client != nil {
-		_, err := s.encoder.Encode(m.ToPoint(s.config.MetaAsTags))
+		_, err := s.encoder.Encode(m.ToPoint(s.meta_as_tags))
 		if err != nil {
 			cclog.ComponentError(s.name, "Write:", err.Error())
 			return err
@@ -105,3 +81,36 @@ func (s *NatsSink) Close() {
 		s.client.Close()
 	}
 }
+
+func NewNatsSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(NatsSink)
+	s.name = fmt.Sprintf("NatsSink(%s)", name)
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &s.config)
+		if err != nil {
+			cclog.ComponentError(s.name, "Error reading config for", s.name, ":", err.Error())
+			return nil, err
+		}
+	}
+	if len(s.config.Host) == 0 ||
+		len(s.config.Port) == 0 ||
+		len(s.config.Database) == 0 {
+		return nil, errors.New("not all configuration variables set required by NatsSink")
+	}
+	// Create lookup map to use meta infos as tags in the output metric
+	s.meta_as_tags = make(map[string]bool)
+	for _, k := range s.config.MetaAsTags {
+		s.meta_as_tags[k] = true
+	}
+	// Setup Influx line protocol
+	s.buffer = &bytes.Buffer{}
+	s.buffer.Grow(1025)
+	s.encoder = influx.NewEncoder(s.buffer)
+	s.encoder.SetPrecision(time.Second)
+	s.encoder.SetMaxLineBytes(1024)
+	// Setup infos for connection
+	if err := s.connect(); err != nil {
+		return nil, fmt.Errorf("unable to connect: %v", err)
+	}
+	return s, nil
+}

sinks/prometheusSink.go

@@ -0,0 +1,199 @@
+package sinks
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+type PrometheusSinkConfig struct {
+	defaultSinkConfig
+	Host             string `json:"host,omitempty"`
+	Port             string `json:"port"`
+	Path             string `json:"path,omitempty"`
+	GroupAsNameSpace bool   `json:"group_as_namespace,omitempty"`
+	// User       string `json:"user,omitempty"`
+	// Password   string `json:"password,omitempty"`
+	// FlushDelay string `json:"flush_delay,omitempty"`
+}
+
+type PrometheusSink struct {
+	sink
+	config       PrometheusSinkConfig
+	labelMetrics map[string]*prometheus.GaugeVec
+	nodeMetrics  map[string]prometheus.Gauge
+	promWg       sync.WaitGroup
+	promServer   *http.Server
+}
+
+func intToFloat64(input interface{}) (float64, error) {
+	switch value := input.(type) {
+	case float64:
+		return value, nil
+	case float32:
+		return float64(value), nil
+	case int:
+		return float64(value), nil
+	case int32:
+		return float64(value), nil
+	case int64:
+		return float64(value), nil
+	}
+	return 0, errors.New("cannot cast value to float64")
+}
+
+func getLabelValue(metric lp.CCMetric) []string {
+	labelValues := []string{}
+	if tid, tidok := metric.GetTag("type-id"); tidok && metric.HasTag("type") {
+		labelValues = append(labelValues, tid)
+	}
+	if d, ok := metric.GetTag("device"); ok {
+		labelValues = append(labelValues, d)
+	} else if d, ok := metric.GetMeta("device"); ok {
+		labelValues = append(labelValues, d)
+	}
+	return labelValues
+}
+
+func getLabelNames(metric lp.CCMetric) []string {
+	labelNames := []string{}
+	if t, tok := metric.GetTag("type"); tok && metric.HasTag("type-id") {
+		labelNames = append(labelNames, t)
+	}
+	if _, ok := metric.GetTag("device"); ok {
+		labelNames = append(labelNames, "device")
+	} else if _, ok := metric.GetMeta("device"); ok {
+		labelNames = append(labelNames, "device")
+	}
+	return labelNames
+}
+
+func (s *PrometheusSink) newMetric(metric lp.CCMetric) error {
+	var value float64 = 0
+	name := metric.Name()
+	opts := prometheus.GaugeOpts{
+		Name: name,
+	}
+	labels := getLabelNames(metric)
+	labelValues := getLabelValue(metric)
+	if len(labels) > 0 && len(labels) != len(labelValues) {
+		return fmt.Errorf("cannot detect metric labels for metric %s", name)
+	}
+
+	if metricValue, ok := metric.GetField("value"); ok {
+		if floatValue, err := intToFloat64(metricValue); err == nil {
+			value = floatValue
+		} else {
+			return fmt.Errorf("metric %s with value '%v' cannot be casted to float64", name, metricValue)
+		}
+	}
+	if s.config.GroupAsNameSpace && metric.HasMeta("group") {
+		g, _ := metric.GetMeta("group")
+		opts.Namespace = strings.ToLower(g)
+	}
+
+	if len(labels) > 0 {
+		new := prometheus.NewGaugeVec(opts, labels)
+		new.WithLabelValues(labelValues...).Set(value)
+		s.labelMetrics[name] = new
+		prometheus.Register(new)
+	} else {
+		new := prometheus.NewGauge(opts)
+		new.Set(value)
+		s.nodeMetrics[name] = new
+		prometheus.Register(new)
+	}
+	return nil
+}
+
+func (s *PrometheusSink) updateMetric(metric lp.CCMetric) error {
+	var value float64 = 0.0
+	name := metric.Name()
+	labelValues := getLabelValue(metric)
+
+	if metricValue, ok := metric.GetField("value"); ok {
+		if floatValue, err := intToFloat64(metricValue); err == nil {
+			value = floatValue
+		} else {
+			return fmt.Errorf("metric %s with value '%v' cannot be casted to float64", name, metricValue)
+		}
+	}
+
+	if len(labelValues) > 0 {
+		if _, ok := s.labelMetrics[name]; !ok {
+			err := s.newMetric(metric)
+			if err != nil {
+				return err
+			}
+		}
+		s.labelMetrics[name].WithLabelValues(labelValues...).Set(value)
+	} else {
+		if _, ok := s.labelMetrics[name]; !ok {
+			err := s.newMetric(metric)
+			if err != nil {
+				return err
+			}
+		}
+		s.nodeMetrics[name].Set(value)
+	}
+	return nil
+}
+
+func (s *PrometheusSink) Write(m lp.CCMetric) error {
+	return s.updateMetric(m)
+}
+
+func (s *PrometheusSink) Flush() error {
+	return nil
+}
+
+func (s *PrometheusSink) Close() {
+	cclog.ComponentDebug(s.name, "CLOSE")
+	s.promServer.Shutdown(context.Background())
+	s.promWg.Wait()
+}
+
+func NewPrometheusSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(PrometheusSink)
+	s.name = "PrometheusSink"
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &s.config)
+		if err != nil {
+			cclog.ComponentError(s.name, "Error reading config for", s.name, ":", err.Error())
+			return nil, err
+		}
+	}
+	if len(s.config.Port) == 0 {
+		err := errors.New("not all configuration variables set required by PrometheusSink")
+		cclog.ComponentError(s.name, err.Error())
+		return nil, err
+	}
+	s.labelMetrics = make(map[string]*prometheus.GaugeVec)
+	s.nodeMetrics = make(map[string]prometheus.Gauge)
+	s.promWg.Add(1)
+	go func() {
+		router := mux.NewRouter()
+		// Prometheus endpoint
+		router.Path("/" + s.config.Path).Handler(promhttp.Handler())
+
+		url := fmt.Sprintf("%s:%s", s.config.Host, s.config.Port)
+		cclog.ComponentDebug(s.name, "Serving Prometheus metrics at", fmt.Sprintf("%s:%s/%s", s.config.Host, s.config.Port, s.config.Path))
+		s.promServer = &http.Server{Addr: url, Handler: router}
+		err := s.promServer.ListenAndServe()
+		if err != nil && err.Error() != "http: Server closed" {
+			cclog.ComponentError(s.name, err.Error())
+		}
+		s.promWg.Done()
+	}()
+	return s, nil
+}

sinks/prometheusSink.md

@@ -0,0 +1,23 @@
+## `prometheus` sink
+
+The `prometheus` sink publishes all metrics via an HTTP server ready to be scraped by a [Prometheus](https://prometheus.io) server. It creates gauge metrics for all node metrics and gauge vectors for all metrics with a subtype like 'device', 'cpu' or 'socket'. 
+
+
+### Configuration structure
+
+```json
+{
+  "<name>": {
+    "type": "prometheus",
+    "host": "localhost",
+    "port": "8080",
+    "path": "metrics"
+  }
+}
+```
+
+- `type`: makes the sink an `prometheus` sink
+- `host`: The HTTP server gets bound to that IP/hostname
+- `port`: Portnumber (as string) for the HTTP server
+- `path`: Path where the metrics should be servered. The metrics will be published at `host`:`port`/`path`
+- `group_as_namespace`: Most metrics contain a group as meta information like 'memory', 'load'. With this the metric names are extended to `group`_`name` if possible.

sinks/sampleSink.go

@@ -0,0 +1,80 @@
+package sinks
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+)
+
+type SampleSinkConfig struct {
+	// defines JSON tags for 'type' and 'meta_as_tags' (string list)
+	// See: metricSink.go
+	defaultSinkConfig
+	// Additional config options, for SampleSink
+}
+
+type SampleSink struct {
+	// declares elements 	'name' and 'meta_as_tags' (string to bool map!)
+	sink
+	config SampleSinkConfig // entry point to the SampleSinkConfig
+}
+
+// Implement functions required for Sink interface
+// Write(...), Flush(), Close()
+// See: metricSink.go
+
+// Code to submit a single CCMetric to the sink
+func (s *SampleSink) Write(point lp.CCMetric) error {
+	// based on s.meta_as_tags use meta infos as tags
+	log.Print(point)
+	return nil
+}
+
+// If the sink uses batched sends internally, you can tell to flush its buffers
+func (s *SampleSink) Flush() error {
+	return nil
+}
+
+// Close sink: close network connection, close files, close libraries, ...
+func (s *SampleSink) Close() {
+	cclog.ComponentDebug(s.name, "CLOSE")
+}
+
+// New function to create a new instance of the sink
+// Initialize the sink by giving it a name and reading in the config JSON
+func NewSampleSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(SampleSink)
+
+	// Set name of sampleSink
+	// The name should be chosen in such a way that different instances of SampleSink can be distinguished
+	s.name = fmt.Sprintf("SampleSink(%s)", name) // Always specify a name here
+
+	// Set defaults in s.config
+	// Allow overwriting these defaults by reading config JSON
+
+	// Read in the config JSON
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &s.config)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Create lookup map to use meta infos as tags in the output metric
+	s.meta_as_tags = make(map[string]bool)
+	for _, k := range s.config.MetaAsTags {
+		s.meta_as_tags[k] = true
+	}
+
+	// Check if all required fields in the config are set
+	// E.g. use 'len(s.config.Option) > 0' for string settings
+
+	// Establish connection to the server, library, ...
+	// Check required files exist and lookup path(s) of executable(s)
+
+	// Return (nil, meaningful error message) in case of errors
+	return s, nil
+}

sinks/sinkManager.go

@@ -13,14 +13,14 @@ import (
 const SINK_MAX_FORWARD = 50
 
 // Map of all available sinks
-var AvailableSinks = map[string]Sink{
-	"influxdb":    new(InfluxSink),
-	"stdout":      new(StdoutSink),
-	"nats":        new(NatsSink),
-	"http":        new(HttpSink),
-	"ganglia":     new(GangliaSink),
-	"influxasync": new(InfluxAsyncSink),
-	"libganglia":  new(LibgangliaSink),
+var AvailableSinks = map[string]func(name string, config json.RawMessage) (Sink, error){
+	"ganglia":     NewGangliaSink,
+	"libganglia":  NewLibgangliaSink,
+	"stdout":      NewStdoutSink,
+	"nats":        NewNatsSink,
+	"influxdb":    NewInfluxSink,
+	"influxasync": NewInfluxAsyncSink,
+	"http":        NewHttpSink,
 }
 
 // Metric collector manager data structure
@@ -149,8 +149,7 @@ func (sm *sinkManager) AddOutput(name string, rawConfig json.RawMessage) error {
 		cclog.ComponentError("SinkManager", "SKIP", name, "unknown sink:", sinkConfig.Type)
 		return err
 	}
-	s := AvailableSinks[sinkConfig.Type]
-	err = s.Init(rawConfig)
+	s, err := AvailableSinks[sinkConfig.Type](name, rawConfig)
 	if err != nil {
 		cclog.ComponentError("SinkManager", "SKIP", s.Name(), "initialization failed:", err.Error())
 		return err

sinks/stdoutSink.go

@@ -19,34 +19,6 @@ type StdoutSink struct {
 	}
 }
 
-func (s *StdoutSink) Init(config json.RawMessage) error {
-	s.name = "StdoutSink"
-	if len(config) > 0 {
-		err := json.Unmarshal(config, &s.config)
-		if err != nil {
-			return err
-		}
-	}
-
-	s.output = os.Stdout
-	if len(s.config.Output) > 0 {
-		switch strings.ToLower(s.config.Output) {
-		case "stdout":
-			s.output = os.Stdout
-		case "stderr":
-			s.output = os.Stderr
-		default:
-			f, err := os.OpenFile(s.config.Output, os.O_CREATE|os.O_WRONLY, os.FileMode(0600))
-			if err != nil {
-				return err
-			}
-			s.output = f
-		}
-	}
-	s.meta_as_tags = s.config.MetaAsTags
-	return nil
-}
-
 func (s *StdoutSink) Write(m lp.CCMetric) error {
 	fmt.Fprint(
 		s.output,
@@ -65,3 +37,37 @@ func (s *StdoutSink) Close() {
 		s.output.Close()
 	}
 }
+
+func NewStdoutSink(name string, config json.RawMessage) (Sink, error) {
+	s := new(StdoutSink)
+	s.name = fmt.Sprintf("StdoutSink(%s)", name)
+	if len(config) > 0 {
+		err := json.Unmarshal(config, &s.config)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	s.output = os.Stdout
+	if len(s.config.Output) > 0 {
+		switch strings.ToLower(s.config.Output) {
+		case "stdout":
+			s.output = os.Stdout
+		case "stderr":
+			s.output = os.Stderr
+		default:
+			f, err := os.OpenFile(s.config.Output, os.O_CREATE|os.O_WRONLY, os.FileMode(0600))
+			if err != nil {
+				return nil, err
+			}
+			s.output = f
+		}
+	}
+	// Create lookup map to use meta infos as tags in the output metric
+	s.meta_as_tags = make(map[string]bool)
+	for _, k := range s.config.MetaAsTags {
+		s.meta_as_tags[k] = true
+	}
+
+	return s, nil
+}