Set proper user for files

Add Red Hat Universal Base Image 8 Workflow

.github/workflows/AlmaLinux.yml

@@ -0,0 +1,64 @@
+# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+
+# Workflow name
+name: AlmaLinux 8.5 RPM build
+
+# Run on tag push
+on:
+ push:
+  tags:
+  - '**'
+
+jobs:
+
+  #
+  # Build on AlmaLinux 8.5 using go-toolset
+  #
+  AlmaLinux-RPM-build:
+    runs-on: ubuntu-latest
+    # See: https://hub.docker.com/_/almalinux
+    container: almalinux:8.5
+    steps:
+
+    # Use dnf to install development packages
+    - name: Install development packages
+      run: dnf --assumeyes group install "Development Tools" "RPM Development Tools"
+
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    # Use dnf to install build dependencies
+    - name: Install build dependencies
+      run: dnf --assumeyes builddep scripts/cc-metric-collector.spec
+
+    - name: RPM build MetricCollector
+      id: rpmbuild
+      run: make RPM
+
+    # See: https://github.com/actions/upload-artifact
+    - name: Save RPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-collector RPM for AlmaLinux 8.5
+        path: ${{ steps.rpmbuild.outputs.RPM }}
+    - name: Save SRPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-collector SRPM for AlmaLinux 8.5
+        path: ${{ steps.rpmbuild.outputs.SRPM }}
+
+    # See: https://github.com/softprops/action-gh-release
+    - name: Release
+      uses: softprops/action-gh-release@v1
+      if: startsWith(github.ref, 'refs/tags/')
+      with:
+        name: cc-metric-collector-${{github.ref_name}}
+        files: |
+         ${{ steps.rpmbuild.outputs.RPM }}
+         ${{ steps.rpmbuild.outputs.SRPM }}

.github/workflows/RedHatUniversalBaseImage.yml

@@ -0,0 +1,64 @@
+# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+
+# Workflow name
+name: Red Hat Universal Base Image 8 RPM build
+
+# Run on tag push
+on:
+ push:
+  tags:
+  - '**'
+
+jobs:
+
+  #
+  # Build on UBI 8 using go-toolset
+  #
+  UBI-8-RPM-build:
+    runs-on: ubuntu-latest
+    # See: https://catalog.redhat.com/software/containers/ubi8/ubi/5c359854d70cc534b3a3784e?container-tabs=gti
+    container: registry.access.redhat.com/ubi8/ubi:8.5-226.1645809065
+    steps:
+
+    # Use dnf to install development packages
+    - name: Install development packages
+      run: dnf --assumeyes --disableplugin=subscription-manager install rpm-build go-srpm-macros rpm-build-libs rpm-libs gcc make python38 git
+
+    # Checkout git repository and submodules
+    # fetch-depth must be 0 to use git describe
+    # See: https://github.com/marketplace/actions/checkout
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    # Use dnf to install build dependencies
+    - name: Install build dependencies
+      run: dnf --assumeyes --disableplugin=subscription-manager builddep scripts/cc-metric-collector.spec
+
+    - name: RPM build MetricCollector
+      id: rpmbuild
+      run: make RPM
+
+    # See: https://github.com/actions/upload-artifact
+    - name: Save RPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-collector RPM for UBI 8
+        path: ${{ steps.rpmbuild.outputs.RPM }}
+    - name: Save SRPM as artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: cc-metric-collector SRPM for UBI 8
+        path: ${{ steps.rpmbuild.outputs.SRPM }}
+
+    # See: https://github.com/softprops/action-gh-release
+    - name: Release
+      uses: softprops/action-gh-release@v1
+      if: startsWith(github.ref, 'refs/tags/')
+      with:
+        name: cc-metric-collector-${{github.ref_name}}
+        files: |
+         ${{ steps.rpmbuild.outputs.RPM }}
+         ${{ steps.rpmbuild.outputs.SRPM }}

.github/workflows/rpmbuild.yml

@@ -1,58 +0,0 @@
-name: Run RPM Build
-on:
- push:
-  tags:
-  - '**'
-
-jobs:
-  build-alma-8_5:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: TomTheBear/rpmbuild@alma8.5
-      id: rpm
-      name: Build RPM package on AlmaLinux 8.5
-      with:
-        spec_file: "./scripts/cc-metric-collector.spec"
-    - name: Save RPM as artifact
-      uses: actions/upload-artifact@v1.0.0
-      with:
-        name: cc-metric-collector RPM AlmaLinux 8.5
-        path: ${{ steps.rpm.outputs.rpm_path }}
-    - name: Save SRPM as artifact
-      uses: actions/upload-artifact@v1.0.0
-      with:
-        name: cc-metric-collector SRPM AlmaLinux 8.5
-        path: ${{ steps.rpm.outputs.source_rpm_path }}
-    - name: Release
-      uses: softprops/action-gh-release@v1
-      with:
-        name: cc-metric-collector-${{github.ref_name}}
-        files: |
-         ${{ steps.rpm.outputs.source_rpm_path }}
-         ${{ steps.rpm.outputs.rpm_path }}
-  build-rhel-ubi8:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: TomTheBear/rpmbuild@rh-ubi8
-      id: rpm
-      name: Build RPM package on Red Hat Universal Base Image 8
-      with:
-        spec_file: "./scripts/cc-metric-collector.spec"
-    - name: Save RPM as artifact
-      uses: actions/upload-artifact@v1.0.0
-      with:
-        name: cc-metric-collector RPM Red Hat Universal Base Image 8
-        path: ${{ steps.rpm.outputs.rpm_path }}
-    - name: Save SRPM as artifact
-      uses: actions/upload-artifact@v1.0.0
-      with:
-        name: cc-metric-collector SRPM Red Hat Universal Base Image 8
-        path: ${{ steps.rpm.outputs.source_rpm_path }}
-    - name: Release
-      uses: softprops/action-gh-release@v1
-      with:
-        files: |
-         ${{ steps.rpm.outputs.source_rpm_path }}
-         ${{ steps.rpm.outputs.rpm_path }}

.github/workflows/runonce.yml

@@ -1,46 +1,68 @@
+# See: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+
+# Workflow name
 name: Run Test
+
+# Run on event push
 on: push
 
 jobs:
+  #
+  # Job build-1-17
+  # Build on latest Ubuntu using golang version 1.17 
+  #
   build-1-17:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    # See: https://github.com/marketplace/actions/checkout
+    # Checkout git repository and submodules
+    - name: Checkout
+      uses: actions/checkout@v2
       with:
         submodules: recursive
 
     # See: https://github.com/marketplace/actions/setup-go-environment
     - name: Setup Golang
-      uses: actions/setup-go@v2.1.5
+      uses: actions/setup-go@v2
       with:
-        go-version: '^1.17.6'
+        go-version: '^1.17.7'
 
+    # Install libganglia
     - name: Setup Ganglia
       run: sudo apt install ganglia-monitor libganglia1
 
     - name: Build MetricCollector
       run: make
 
-    - name: Run MetricCollector
+    - name: Run MetricCollector once
       run: ./cc-metric-collector --once --config .github/ci-config.json
+
+  #
+  # Job build-1-16
+  # Build on latest Ubuntu using golang version 1.16
+  #
   build-1-16:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    # See: https://github.com/marketplace/actions/checkout
+    # Checkout git repository and submodules
+    - name: Checkout
+      uses: actions/checkout@v2
       with:
         submodules: recursive
 
     # See: https://github.com/marketplace/actions/setup-go-environment
     - name: Setup Golang
-      uses: actions/setup-go@v2.1.5
+      uses: actions/setup-go@v2
       with:
         go-version: '^1.16.7' # The version AlmaLinux 8.5 uses
 
+    # Install libganglia
     - name: Setup Ganglia
       run: sudo apt install ganglia-monitor libganglia1
 
     - name: Build MetricCollector
       run: make
 
-    - name: Run MetricCollectorlibganglia1
+    - name: Run MetricCollector once
       run: ./cc-metric-collector --once --config .github/ci-config.json

Makefile

@@ -1,5 +1,5 @@
 APP = cc-metric-collector
-GOSRC_APP        := metric-collector.go
+GOSRC_APP        := cc-metric-collector.go
 GOSRC_COLLECTORS := $(wildcard collectors/*.go)
 GOSRC_SINKS      := $(wildcard sinks/*.go)
 GOSRC_RECEIVERS  := $(wildcard receivers/*.go)
@@ -56,3 +56,33 @@ vet:
 staticcheck:
 	go install honnef.co/go/tools/cmd/staticcheck@latest
 	$$(go env GOPATH)/bin/staticcheck ./...
+
+.ONESHELL:
+.PHONY: RPM
+RPM: scripts/cc-metric-collector.spec
+	@WORKSPACE="$${PWD}"
+	@SPECFILE="$${WORKSPACE}/scripts/cc-metric-collector.spec"
+	# Setup RPM build tree
+	@eval $$(rpm --eval "ARCH='%{_arch}' RPMDIR='%{_rpmdir}' SOURCEDIR='%{_sourcedir}' SPECDIR='%{_specdir}' SRPMDIR='%{_srcrpmdir}' BUILDDIR='%{_builddir}'")
+	@mkdir --parents --verbose "$${RPMDIR}" "$${SOURCEDIR}" "$${SPECDIR}" "$${SRPMDIR}" "$${BUILDDIR}"
+	# Create source tarball
+	@COMMITISH="HEAD"
+	@VERS=$$(git describe --tags $${COMMITISH})
+	@VERS=$${VERS#v}
+	@VERS=$${VERS//-/_}
+	@eval $$(rpmspec --query --queryformat "NAME='%{name}' VERSION='%{version}' RELEASE='%{release}' NVR='%{NVR}' NVRA='%{NVRA}'" --define="VERS $${VERS}" "$${SPECFILE}")
+	@PREFIX="$${NAME}-$${VERSION}"
+	@FORMAT="tar.gz"
+	@SRCFILE="$${SOURCEDIR}/$${PREFIX}.$${FORMAT}"
+	@git archive --verbose --format "$${FORMAT}" --prefix="$${PREFIX}/" --output="$${SRCFILE}" $${COMMITISH}
+	# Build RPM and SRPM
+	@rpmbuild -ba --define="VERS $${VERS}" --rmsource --clean "$${SPECFILE}"
+	# Report RPMs and SRPMs when in GitHub Workflow
+	@if [[ "$${GITHUB_ACTIONS}" == true ]]; then
+	@     RPMFILE="$${RPMDIR}/$${ARCH}/$${NVRA}.rpm"
+	@     SRPMFILE="$${SRPMDIR}/$${NVR}.src.rpm"
+	@     echo "RPM: $${RPMFILE}"
+	@     echo "SRPM: $${SRPMFILE}"
+	@     echo "::set-output name=SRPM::$${SRPMFILE}"
+	@     echo "::set-output name=RPM::$${RPMFILE}"
+	@fi

collectors/Makefile

@@ -55,16 +55,13 @@ $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION): $(BUILD_FOLDER)/likwid-$(LIKWID_VERSIO
 	tar -C $(BUILD_FOLDER) -xf $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION).tar.gz
 
 $(INSTALL_FOLDER)/liblikwid.a: $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION) $(INSTALL_FOLDER)
-	sed -i -e s+"PREFIX ?= .*"+"PREFIX = $(LIKWID_BASE)"+g \
-		-e s+"SHARED_LIBRARY = .*"+"SHARED_LIBRARY = false"+g \
-		-e s+"ACCESSMODE = .*"+"ACCESSMODE = $(ACCESSMODE)"+g \
-		-e s+"INSTALLED_ACCESSDAEMON = .*"+"INSTALLED_ACCESSDAEMON = $(DAEMON_INSTALLDIR)/likwid-accessD"+g \
-		$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/config.mk
-	cd $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION) && make
-	cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/liblikwid.a $(INSTALL_FOLDER)
-	cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/ext/hwloc/liblikwid-hwloc.a $(INSTALL_FOLDER)
-	cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/src/includes/likwid*.h $(INSTALL_FOLDER)
-	cp $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/src/includes/bstrlib.h $(INSTALL_FOLDER)
+	cd "$(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)" && make "PREFIX=$(LIKWID_BASE)" "SHARED_LIBRARY=false" "ACCESSMODE=$(ACCESSMODE)" "INSTALLED_ACCESSDAEMON=$(DAEMON_INSTALLDIR)/likwid-accessD"
+	cp \
+	    $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/liblikwid.a \
+	    $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/ext/hwloc/liblikwid-hwloc.a \
+	    $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/src/includes/likwid*.h \
+	    $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/src/includes/bstrlib.h \
+	    $(INSTALL_FOLDER)
 
 $(DAEMON_INSTALLDIR)/likwid-accessD: $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/likwid-accessD
 	sudo -u $(DAEMON_USER) -g $(DAEMON_GROUP) install -m 4775 $(BUILD_FOLDER)/likwid-$(LIKWID_VERSION)/likwid-accessD $(DAEMON_INSTALLDIR)/likwid-accessD

collectors/likwidMetric.go

@@ -2,7 +2,7 @@ package collectors
 
 /*
 #cgo CFLAGS: -I./likwid
-#cgo LDFLAGS: -L./likwid -llikwid -llikwid-hwloc -lm -Wl,--unresolved-symbols=ignore-in-object-files
+#cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files
 #include <stdlib.h>
 #include <likwid.h>
 */
@@ -71,8 +71,9 @@ func GetAllMetricScopes() []MetricScope {
 }
 
 const (
-	LIKWID_LIB_NAME     = "liblikwid.so"
-	LIKWID_LIB_DL_FLAGS = dl.RTLD_LAZY | dl.RTLD_GLOBAL
+	LIKWID_LIB_NAME       = "liblikwid.so"
+	LIKWID_LIB_DL_FLAGS   = dl.RTLD_LAZY | dl.RTLD_GLOBAL
+	LIKWID_DEF_ACCESSMODE = "direct"
 )
 
 type LikwidCollectorMetricConfig struct {
@@ -95,6 +96,8 @@ type LikwidCollectorConfig struct {
 	Metrics        []LikwidCollectorMetricConfig   `json:"globalmetrics,omitempty"`
 	ForceOverwrite bool                            `json:"force_overwrite,omitempty"`
 	InvalidToZero  bool                            `json:"invalid_to_zero,omitempty"`
+	AccessMode     string                          `json:"access_mode,omitempty"`
+	DaemonPath     string                          `json:"accessdaemon_path,omitempty"`
 }
 
 type LikwidCollector struct {
@@ -260,6 +263,7 @@ func (m *LikwidCollector) getResponsiblities() map[MetricScope]map[int]int {
 func (m *LikwidCollector) Init(config json.RawMessage) error {
 	var ret C.int
 	m.name = "LikwidCollector"
+	m.config.AccessMode = LIKWID_DEF_ACCESSMODE
 	if len(config) > 0 {
 		err := json.Unmarshal(config, &m.config)
 		if err != nil {
@@ -270,6 +274,11 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 	if lib == nil {
 		return fmt.Errorf("error instantiating DynamicLibrary for %s", LIKWID_LIB_NAME)
 	}
+	err := lib.Open()
+	if err != nil {
+		return fmt.Errorf("error opening %s: %v", LIKWID_LIB_NAME, err)
+	}
+
 	if m.config.ForceOverwrite {
 		cclog.ComponentDebug(m.name, "Set LIKWID_FORCE=1")
 		os.Setenv("LIKWID_FORCE", "1")
@@ -301,6 +310,16 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 	m.initGranularity()
 	// Generate map for MetricScope -> scope_id (like socket id) -> responsible id (offset in cpulist)
 	m.scopeRespTids = m.getResponsiblities()
+	switch m.config.AccessMode {
+	case "direct":
+		C.HPMmode(0)
+	case "accessdaemon":
+		if len(m.config.DaemonPath) > 0 {
+			p := os.Getenv("PATH")
+			os.Setenv("PATH", m.config.DaemonPath+":"+p)
+		}
+		C.HPMmode(1)
+	}
 
 	cclog.ComponentDebug(m.name, "initialize LIKWID perfmon module")
 	ret = C.perfmon_init(C.int(len(m.cpulist)), &m.cpulist[0])

collectors/likwidMetric.md

@@ -8,6 +8,8 @@ The `likwid` configuration consists of two parts, the "eventsets" and "globalmet
 - The global metrics are metrics which require data from all event set measurements to be derived. The inputs are the metrics in the event sets. Similar to the metrics in the event sets, the global metrics are defined by a name, a formula, a scope and a publish flag. See event set metrics for details. The only difference is that there is no access to the raw event measurements anymore but only to the metrics. So, the idea is to derive a metric in the "eventsets" section and reuse it in the "globalmetrics" part. If you need a metric only for deriving the global metrics, disable forwarding of the event set metrics. **Be aware** that the combination might be misleading because the "behavior" of a metric changes over time and the multiple measurements might count different computing phases.
 
 Additional options:
+- `access_mode` : Method to use for hardware performance monitoring (`direct` access as root user, `accessdaemon` for the daemon mode)
+- `accessdaemon_path`: Folder with the access daemon `likwid-accessD`, commonly `$LIKWID_INSTALL_LOC/sbin`
 - `force_overwrite`: Same as setting `LIKWID_FORCE=1`. In case counters are already in-use, LIKWID overwrites their configuration to do its measurements
 - `invalid_to_zero`: In some cases, the calculations result in `NaN` or `Inf`. With this option, all `NaN` and `Inf` values are replaces with `0.0`.
 
@@ -61,7 +63,21 @@ $ scripts/likwid_perfgroup_to_cc_config.py ICX MEM_DP
 }
 ```
 
-You can copy this JSON and add it to the `eventsets` list. If you specify multiple event sets, you can add globally derived metrics in the extra `global_metrics` section with the metric names as variables. 
+You can copy this JSON and add it to the `eventsets` list. If you specify multiple event sets, you can add globally derived metrics in the extra `global_metrics` section with the metric names as variables.
+
+### Mixed usage between daemon and users
+
+LIKWID checks the file `/var/run/likwid.lock` before performing any interfering operations. Who is allowed to access the counters is determined by the owner of the file. If it does not exist, it is created for the current user. So, if you want to temporarly allow counter access to a user (e.g. in a job):
+
+Before (SLURM prolog, ...)
+```
+$ chwon $JOBUSER /var/run/likwid.lock
+```
+
+After (SLURM epilog, ...)
+```
+$ chwon $CCUSER /var/run/likwid.lock
+```
 
 ### Example configuration
 

collectors/lustreMetric.go

@@ -22,6 +22,7 @@ type LustreCollectorConfig struct {
 	LCtlCommand    string   `json:"lctl_command"`
 	ExcludeMetrics []string `json:"exclude_metrics"`
 	SendAllMetrics bool     `json:"send_all_metrics"`
+	Sudo           bool     `json:"use_sudo"`
 }
 
 type LustreCollector struct {
@@ -31,11 +32,17 @@ type LustreCollector struct {
 	stats   map[string]map[string]int64
 	config  LustreCollectorConfig
 	lctl    string
+	sudoCmd string
 }
 
 func (m *LustreCollector) getDeviceDataCommand(device string) []string {
+	var command *exec.Cmd
 	statsfile := fmt.Sprintf("llite.%s.stats", device)
-	command := exec.Command(m.lctl, LCTL_OPTION, statsfile)
+	if m.config.Sudo {
+		command = exec.Command(m.sudoCmd, m.lctl, LCTL_OPTION, statsfile)
+	} else {
+		command = exec.Command(m.lctl, LCTL_OPTION, statsfile)
+	}
 	command.Wait()
 	stdout, _ := command.Output()
 	return strings.Split(string(stdout), "\n")
@@ -109,7 +116,7 @@ func (m *LustreCollector) Init(config json.RawMessage) error {
 		return err
 	}
 	if user.Uid != "0" {
-		cclog.ComponentError(m.name, "Lustre file system statistics can only be queried by user root:", err.Error())
+		cclog.ComponentError(m.name, "Lustre file system statistics can only be queried by user root")
 		return err
 	}
 
@@ -136,6 +143,12 @@ func (m *LustreCollector) Init(config json.RawMessage) error {
 		}
 	}
 	m.lctl = p
+	if m.config.Sudo {
+		p, err := exec.LookPath("sudo")
+		if err != nil {
+			m.sudoCmd = p
+		}
+	}
 
 	devices := m.getDevices()
 	if len(devices) == 0 {

collectors/metricCollector.go

@@ -13,29 +13,30 @@ import (
 )
 
 type MetricCollector interface {
-	Name() string
-	Init(config json.RawMessage) error
-	Initialized() bool
-	Read(duration time.Duration, output chan lp.CCMetric)
-	Close()
+	Name() string                                         // Name of the metric collector
+	Init(config json.RawMessage) error                    // Initialize metric collector
+	Initialized() bool                                    // Is metric collector initialized?
+	Read(duration time.Duration, output chan lp.CCMetric) // Read metrics from metric collector
+	Close()                                               // Close / finish metric collector
 }
 
 type metricCollector struct {
-	name string
-	init bool
-	meta map[string]string
+	name string            // name of the metric
+	init bool              // is metric collector initialized?
+	meta map[string]string // static meta data tags
 }
 
-// Name() returns the name of the metric collector
+// Name returns the name of the metric collector
 func (c *metricCollector) Name() string {
 	return c.name
 }
 
+// Setup is for future use
 func (c *metricCollector) setup() error {
 	return nil
 }
 
-// Initialized() indicates whether the metric collector has been initialized.
+// Initialized indicates whether the metric collector has been initialized
 func (c *metricCollector) Initialized() bool {
 	return c.init
 }
@@ -64,6 +65,7 @@ func stringArrayContains(array []string, str string) (int, bool) {
 	return -1, false
 }
 
+// SocketList returns the list of physical sockets as read from /proc/cpuinfo
 func SocketList() []int {
 	buffer, err := ioutil.ReadFile("/proc/cpuinfo")
 	if err != nil {
@@ -89,6 +91,7 @@ func SocketList() []int {
 	return packs
 }
 
+// CpuList returns the list of physical CPUs (in contrast to logical CPUs) as read from /proc/cpuinfo
 func CpuList() []int {
 	buffer, err := ioutil.ReadFile("/proc/cpuinfo")
 	if err != nil {
@@ -117,8 +120,8 @@ func CpuList() []int {
 // RemoveFromStringList removes the string r from the array of strings s
 // If r is not contained in the array an error is returned
 func RemoveFromStringList(s []string, r string) ([]string, error) {
-	for i, item := range s {
-		if r == item {
+	for i := range s {
+		if r == s[i] {
 			return append(s[:i], s[i+1:]...), nil
 		}
 	}

collectors/sampleMetric.go

@@ -22,6 +22,13 @@ type SampleCollector struct {
 	tags   map[string]string          // default tags
 }
 
+// Functions to implement MetricCollector interface
+// Init(...), Read(...), Close()
+// See: metricCollector.go
+
+// Init initializes the sample collector
+// Called once by the collector manager
+// All tags, meta data tags and metrics that do not change over the runtime should be set here
 func (m *SampleCollector) Init(config json.RawMessage) error {
 	var err error = nil
 	// Always set the name early in Init() to use it in cclog.Component* functions
@@ -56,12 +63,14 @@ func (m *SampleCollector) Init(config json.RawMessage) error {
 	return err
 }
 
+// Read collects all metrics belonging to the sample collector
+// and sends them through the output channel to the collector manager
 func (m *SampleCollector) Read(interval time.Duration, output chan lp.CCMetric) {
 	// Create a sample metric
 	timestamp := time.Now()
 
 	value := 1.0
-	// If you want to measure something for a specific amout of time, use interval
+	// If you want to measure something for a specific amount of time, use interval
 	// start := readState()
 	// time.Sleep(interval)
 	// stop := readState()
@@ -75,6 +84,8 @@ func (m *SampleCollector) Read(interval time.Duration, output chan lp.CCMetric)
 
 }
 
+// Close metric collector: close network connection, close files, close libraries, ...
+// Called once by the collector manager
 func (m *SampleCollector) Close() {
 	// Unset flag
 	m.init = false

receivers/README.md

@@ -9,32 +9,20 @@ The configuration file for the receivers is a list of configurations. The `type`
 ```json
 {
   "myreceivername" : {
+    "type": "receiver-type",
     <receiver-specific configuration>
   }
 }
 ```
 
-## Type `nats`
+This allows to specify
 
-```json
-{
-  "type": "nats",
-  "address": "<nats-URI or hostname>",
-  "port" : "<portnumber>",
-  "subject": "<subscribe topic>"
-}
-```
+## Available receivers
 
-The `nats` receiver subscribes to the topic `database` and listens on `address` and `port` for metrics in the InfluxDB line protocol.
+- [`nats`](./natsReceiver.md): Receive metrics from the NATS network
+- [`prometheus`](./prometheusReceiver.md): Scrape data from a Prometheus client
 
 # Contributing own receivers
 A receiver contains a few functions and is derived from the type `Receiver` (in `metricReceiver.go`):
-* `Start() error`
-* `Close()`
-* `Name() string`
-* `SetSink(sink chan lp.CCMetric)`
-* `New<Typename>(name string, config json.RawMessage)`
 
-The data structures should be set up in `Init()` like opening a file or server connection. The `Start()` function should either start a go routine or issue some other asynchronous mechanism for receiving metrics. The `Close()` function should tear down anything created in `Init()`.
-
-Finally, the receiver needs to be registered in the `receiveManager.go`. There is a list of receivers called `AvailableReceivers` which is a map (`receiver_type_string` -> `pointer to NewReceiver function`). Add a new entry with a descriptive name and the new receiver.
+For an example, check the [sample receiver](./sampleReceiver.go)

receivers/metricReceiver.go

@@ -8,6 +8,7 @@ type defaultReceiverConfig struct {
 	Type string `json:"type"`
 }
 
+// Receiver configuration: Listen address, port
 type ReceiverConfig struct {
 	Addr         string `json:"address"`
 	Port         string `json:"port"`
@@ -23,15 +24,17 @@ type receiver struct {
 
 type Receiver interface {
 	Start()
-	Close()
-	Name() string
-	SetSink(sink chan lp.CCMetric)
+	Close()                        // Close / finish metric receiver
+	Name() string                  // Name of the metric receiver
+	SetSink(sink chan lp.CCMetric) // Set sink channel
 }
 
+// Name returns the name of the metric receiver
 func (r *receiver) Name() string {
 	return r.name
 }
 
+// SetSink set the sink channel
 func (r *receiver) SetSink(sink chan lp.CCMetric) {
 	r.sink = sink
 }

receivers/natsReceiver.md

@@ -0,0 +1,21 @@
+## `nats` receiver
+
+The `nats` receiver can be used receive metrics from the NATS network. The `nats` receiver subscribes to the topic `database` and listens on `address` and `port` for metrics in the InfluxDB line protocol.
+
+### Configuration structure
+
+```json
+{
+  "<name>": {
+    "type": "nats",
+    "address" : "nats-server.example.org",
+    "port" : "4222",
+    "subject" : "subject"
+  }
+}
+```
+
+- `type`: makes the receiver a `nats` receiver
+- `address`: Address of the NATS control server
+- `port`: Port of the NATS control server
+- `subject`: Subscribes to this subject and receive metrics

receivers/prometheusReceiver.md

@@ -10,7 +10,7 @@ The `prometheus` receiver can be used to scrape the metrics of a single `prometh
     "type": "prometheus",
     "address" : "testpromhost",
     "port" : "12345",
-    "port" : "/prometheus",
+    "path" : "/prometheus",
     "interval": "5s",
     "ssl" : true,
   }

receivers/sampleReceiver.go

@@ -7,6 +7,7 @@ import (
 	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
 )
 
+// SampleReceiver configuration: receiver type, listen address, port
 type SampleReceiverConfig struct {
 	Type string `json:"type"`
 	Addr string `json:"address"`
@@ -24,6 +25,10 @@ type SampleReceiver struct {
 	// wg   sync.WaitGroup
 }
 
+// Implement functions required for Receiver interface
+// Start(), Close()
+// See: metricReceiver.go
+
 func (r *SampleReceiver) Start() {
 	cclog.ComponentDebug(r.name, "START")
 
@@ -44,6 +49,7 @@ func (r *SampleReceiver) Start() {
 	// }()
 }
 
+// Close receiver: close network connection, close files, close libraries, ...
 func (r *SampleReceiver) Close() {
 	cclog.ComponentDebug(r.name, "CLOSE")
 
@@ -54,13 +60,21 @@ func (r *SampleReceiver) Close() {
 	// r.wg.Wait()
 }
 
+// New function to create a new instance of the receiver
+// Initialize the receiver by giving it a name and reading in the config JSON
 func NewSampleReceiver(name string, config json.RawMessage) (Receiver, error) {
 	r := new(SampleReceiver)
-	r.name = fmt.Sprintf("HttpReceiver(%s)", name)
+
+	// Set name of SampleReceiver
+	// The name should be chosen in such a way that different instances of SampleReceiver can be distinguished
+	r.name = fmt.Sprintf("SampleReceiver(%s)", name)
 
 	// Set static information
 	r.meta = map[string]string{"source": r.name}
 
+	// Set defaults in r.config
+	// Allow overwriting these defaults by reading config JSON
+
 	// Read the sample receiver specific JSON config
 	if len(config) > 0 {
 		err := json.Unmarshal(config, &r.config)

scripts/cc-metric-collector.spec

@@ -1,5 +1,5 @@
 Name:           cc-metric-collector
-Version:        0.2
+Version:        %{VERS}
 Release:        1%{?dist}
 Summary:        Metric collection daemon from the ClusterCockpit suite
 
@@ -7,6 +7,7 @@ License:        MIT
 Source0:        %{name}-%{version}.tar.gz
 
 BuildRequires:  go-toolset
+BuildRequires:  systemd-rpm-macros
 # for internal LIKWID installation
 BuildRequires:  wget perl-Data-Dumper
 
@@ -34,11 +35,15 @@ install -Dpm 0600 receivers.json %{buildroot}%{_sysconfdir}/%{name}/receivers.js
 install -Dpm 0600 router.json %{buildroot}%{_sysconfdir}/%{name}/router.json
 install -Dpm 0644 scripts/%{name}.service %{buildroot}%{_unitdir}/%{name}.service
 install -Dpm 0600 scripts/%{name}.config %{buildroot}%{_sysconfdir}/default/%{name}
+install -Dpm 0644 scripts/%{name}.sysusers %{buildroot}%{_sysusersdir}/%{name}.conf
 
 
 %check
 # go test should be here... :)
 
+%pre
+%sysusers_create_package scripts/%{name}.sysusers
+
 %post
 %systemd_post %{name}.service
 
@@ -46,17 +51,23 @@ install -Dpm 0600 scripts/%{name}.config %{buildroot}%{_sysconfdir}/default/%{na
 %systemd_preun %{name}.service
 
 %files
+# Binary
+%attr(-,clustercockpit,clustercockpit) %{_sbindir}/%{name}
+# Configuration
 %dir %{_sysconfdir}/%{name}
-%{_sbindir}/%{name}
+%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/%{name}.json
+%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/collectors.json
+%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/sinks.json
+%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/receivers.json
+%attr(0600,clustercockpit,clustercockpit) %config(noreplace) %{_sysconfdir}/%{name}/router.json
+# Systemd
+%{_sysusersdir}/%{name}.conf
 %{_unitdir}/%{name}.service
 %{_sysconfdir}/default/%{name}
-%attr(0600,root,root) %config(noreplace) %{_sysconfdir}/%{name}/%{name}.json
-%attr(0600,root,root) %config(noreplace) %{_sysconfdir}/%{name}/collectors.json
-%attr(0600,root,root) %config(noreplace) %{_sysconfdir}/%{name}/sinks.json
-%attr(0600,root,root) %config(noreplace) %{_sysconfdir}/%{name}/receivers.json
-%attr(0600,root,root) %config(noreplace) %{_sysconfdir}/%{name}/router.json
 
 %changelog
+* Thu Mar 03 2022 Thomas Gruber - 0.3
+- Add clustercockpit user installation
 * Mon Feb 14 2022 Thomas Gruber - 0.2
 - Add component specific configuration files
 - Add %attr to config files

scripts/cc-metric-collector.sysusers

@@ -0,0 +1,2 @@
+#Type Name           ID GECOS                     Home directory           Shell
+u     clustercockpit -  "User for ClusterCockpit" /run/cc-metric-collector /sbin/nologin

sinks/metricSink.go

@@ -10,17 +10,18 @@ type defaultSinkConfig struct {
 }
 
 type sink struct {
-	meta_as_tags bool
-	name         string
+	meta_as_tags bool   // Use meta data tags as tags
+	name         string // Name of the sink
 }
 
 type Sink interface {
-	Write(point lp.CCMetric) error
-	Flush() error
-	Close()
-	Name() string
+	Write(point lp.CCMetric) error // Write metric to the sink
+	Flush() error                  // Flush buffered metrics
+	Close()                        // Close / finish metric sink
+	Name() string                  // Name of the metric sink
 }
 
+// Name returns the name of the metric sink
 func (s *sink) Name() string {
 	return s.name
 }

sinks/sampleSink.go

@@ -10,15 +10,22 @@ import (
 )
 
 type SampleSinkConfig struct {
-	defaultSinkConfig // defines JSON tags for 'type' and 'meta_as_tags'
-	// Add additional options
+	// defines JSON tags for 'type' and 'meta_as_tags'
+	// See: metricSink.go
+	defaultSinkConfig
+	// Additional config options, for SampleSink
 }
 
 type SampleSink struct {
-	sink                    // declarate 'name' and 'meta_as_tags'
+	// declares elements 	'name' and 'meta_as_tags'
+	sink
 	config SampleSinkConfig // entry point to the SampleSinkConfig
 }
 
+// Implement functions required for Sink interface
+// Write(...), Flush(), Close()
+// See: metricSink.go
+
 // Code to submit a single CCMetric to the sink
 func (s *SampleSink) Write(point lp.CCMetric) error {
 	log.Print(point)
@@ -39,9 +46,13 @@ func (s *SampleSink) Close() {
 // Initialize the sink by giving it a name and reading in the config JSON
 func NewSampleSink(name string, config json.RawMessage) (Sink, error) {
 	s := new(SampleSink)
+
+	// Set name of sampleSink
+	// The name should be chosen in such a way that different instances of SampleSink can be distinguished
 	s.name = fmt.Sprintf("SampleSink(%s)", name) // Always specify a name here
 
 	// Set defaults in s.config
+	// Allow overwriting these defaults by reading config JSON
 
 	// Read in the config JSON
 	if len(config) > 0 {
@@ -52,7 +63,7 @@ func NewSampleSink(name string, config json.RawMessage) (Sink, error) {
 	}
 
 	// Check if all required fields in the config are set
-	// Use 'len(s.config.Option) > 0' for string settings
+	// E.g. use 'len(s.config.Option) > 0' for string settings
 
 	// Establish connection to the server, library, ...
 	// Check required files exist and lookup path(s) of executable(s)