Add latest development to main branch (#89)

* InfiniBandCollector: Scale raw readings from octets to bytes

* Fix clock frequency coming from LikwidCollector and update docs

* Build DEB package for Ubuntu 20.04 for releases

* Fix memstat collector with numa_stats option

* Remove useless prints from MemstatCollector

* Replace ioutils with os and io (#87)

* Use lower case for error strings in RocmSmiCollector

* move maybe-usable-by-other-cc-components to pkg. Fix all files to use the new paths (#88)

* Add collector for monitoring the execution of cc-metric-collector itself (#81)

* Add collector to monitor execution of cc-metric-collector itself

* Register SelfCollector

* Fix import paths for moved packages

collectors/beegfsmetaMetric.go

@@ -5,7 +5,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"os/exec"
 	"os/user"
@@ -14,8 +14,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 const DEFAULT_BEEGFS_CMD = "beegfs-ctl"
@@ -115,7 +115,7 @@ func (m *BeegfsMetaCollector) Read(interval time.Duration, output chan lp.CCMetr
 		return
 	}
 	//get mounpoint
-	buffer, _ := ioutil.ReadFile(string("/proc/mounts"))
+	buffer, _ := os.ReadFile(string("/proc/mounts"))
 	mounts := strings.Split(string(buffer), "\n")
 	var mountpoints []string
 	for _, line := range mounts {
@@ -157,9 +157,9 @@ func (m *BeegfsMetaCollector) Read(interval time.Duration, output chan lp.CCMetr
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error())
 			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode())
-			data, _ := ioutil.ReadAll(cmdStderr)
+			data, _ := io.ReadAll(cmdStderr)
 			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stderr: \"%s\"\n", string(data))
-			data, _ = ioutil.ReadAll(cmdStdout)
+			data, _ = io.ReadAll(cmdStdout)
 			fmt.Fprintf(os.Stderr, "BeegfsMetaCollector.Read(): command stdout: \"%s\"\n", string(data))
 			return
 		}

collectors/beegfsstorageMetric.go

@@ -5,7 +5,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"os/exec"
 	"os/user"
@@ -14,8 +14,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 // Struct for the collector-specific JSON config
@@ -108,7 +108,7 @@ func (m *BeegfsStorageCollector) Read(interval time.Duration, output chan lp.CCM
 		return
 	}
 	//get mounpoint
-	buffer, _ := ioutil.ReadFile(string("/proc/mounts"))
+	buffer, _ := os.ReadFile(string("/proc/mounts"))
 	mounts := strings.Split(string(buffer), "\n")
 	var mountpoints []string
 	for _, line := range mounts {
@@ -149,9 +149,9 @@ func (m *BeegfsStorageCollector) Read(interval time.Duration, output chan lp.CCM
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): Failed to execute command \"%s\": %s\n", cmd.String(), err.Error())
 			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command exit code: \"%d\"\n", cmd.ProcessState.ExitCode())
-			data, _ := ioutil.ReadAll(cmdStderr)
+			data, _ := io.ReadAll(cmdStderr)
 			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stderr: \"%s\"\n", string(data))
-			data, _ = ioutil.ReadAll(cmdStdout)
+			data, _ = io.ReadAll(cmdStdout)
 			fmt.Fprintf(os.Stderr, "BeegfsStorageCollector.Read(): command stdout: \"%s\"\n", string(data))
 			return
 		}

collectors/collectorManager.go

@@ -6,9 +6,9 @@ import (
 	"sync"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
-	mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
+	mct "github.com/ClusterCockpit/cc-metric-collector/pkg/multiChanTicker"
 )
 
 // Map of all available metric collectors
@@ -37,7 +37,8 @@ var AvailableCollectors = map[string]MetricCollector{
 	"beegfs_meta":     new(BeegfsMetaCollector),
 	"beegfs_storage":  new(BeegfsStorageCollector),
 	"rocm_smi":        new(RocmSmiCollector),
-	"schedstat":	   new(SchedstatCollector),
+	"self":            new(SelfCollector),
+	"schedstat":       new(SchedstatCollector),
 }
 
 // Metric collector manager data structure

collectors/cpufreqCpuinfoMetric.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 //

collectors/cpufreqMetric.go

@@ -3,14 +3,14 @@ package collectors
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 	"golang.org/x/sys/unix"
 )
 
@@ -88,7 +88,7 @@ func (m *CPUFreqCollector) Init(config json.RawMessage) error {
 
 		// Read package ID
 		physicalPackageIDFile := filepath.Join(cpuDir, "topology", "physical_package_id")
-		line, err := ioutil.ReadFile(physicalPackageIDFile)
+		line, err := os.ReadFile(physicalPackageIDFile)
 		if err != nil {
 			return fmt.Errorf("unable to read physical package ID from file '%s': %v", physicalPackageIDFile, err)
 		}
@@ -100,7 +100,7 @@ func (m *CPUFreqCollector) Init(config json.RawMessage) error {
 
 		// Read core ID
 		coreIDFile := filepath.Join(cpuDir, "topology", "core_id")
-		line, err = ioutil.ReadFile(coreIDFile)
+		line, err = os.ReadFile(coreIDFile)
 		if err != nil {
 			return fmt.Errorf("unable to read core ID from file '%s': %v", coreIDFile, err)
 		}
@@ -188,7 +188,7 @@ func (m *CPUFreqCollector) Read(interval time.Duration, output chan lp.CCMetric)
 		}
 
 		// Read current frequency
-		line, err := ioutil.ReadFile(t.scalingCurFreqFile)
+		line, err := os.ReadFile(t.scalingCurFreqFile)
 		if err != nil {
 			cclog.ComponentError(
 				m.name,

collectors/cpustatMetric.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 	sysconf "github.com/tklauser/go-sysconf"
 )
 
@@ -22,12 +22,12 @@ type CpustatCollectorConfig struct {
 
 type CpustatCollector struct {
 	metricCollector
-	config   CpustatCollectorConfig
-	lastTimestamp time.Time                   // Store time stamp of last tick to derive values
-	matches  map[string]int
-	cputags  map[string]map[string]string
-	nodetags map[string]string
-	olddata  map[string]map[string]int64
+	config        CpustatCollectorConfig
+	lastTimestamp time.Time // Store time stamp of last tick to derive values
+	matches       map[string]int
+	cputags       map[string]map[string]string
+	nodetags      map[string]string
+	olddata       map[string]map[string]int64
 }
 
 func (m *CpustatCollector) Init(config json.RawMessage) error {
@@ -118,7 +118,7 @@ func (m *CpustatCollector) parseStatLine(linefields []string, tags map[string]st
 			}
 		}
 	}
-	
+
 	for name, value := range values {
 		y, err := lp.New(name, tags, m.meta, map[string]interface{}{"value": value * 100}, now)
 		if err == nil {

collectors/customCmdMetric.go

@@ -3,13 +3,13 @@ package collectors
 import (
 	"encoding/json"
 	"errors"
-	"io/ioutil"
 	"log"
+	"os"
 	"os/exec"
 	"strings"
 	"time"
 
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 	influx "github.com/influxdata/line-protocol"
 )
 
@@ -53,7 +53,7 @@ func (m *CustomCmdCollector) Init(config json.RawMessage) error {
 		}
 	}
 	for _, f := range m.config.Files {
-		_, err = ioutil.ReadFile(f)
+		_, err = os.ReadFile(f)
 		if err == nil {
 			m.files = append(m.files, f)
 		} else {
@@ -106,7 +106,7 @@ func (m *CustomCmdCollector) Read(interval time.Duration, output chan lp.CCMetri
 		}
 	}
 	for _, file := range m.files {
-		buffer, err := ioutil.ReadFile(file)
+		buffer, err := os.ReadFile(file)
 		if err != nil {
 			log.Print(err)
 			return

collectors/diskstatMetric.go

@@ -8,8 +8,8 @@ import (
 	"syscall"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 //	"log"

collectors/gpfsMetric.go

@@ -5,7 +5,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"os/exec"
 	"os/user"
@@ -13,8 +13,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 const DEFAULT_GPFS_CMD = "mmpmon"
@@ -118,8 +118,8 @@ func (m *GpfsCollector) Read(interval time.Duration, output chan lp.CCMetric) {
 	cmd.Stderr = cmdStderr
 	err := cmd.Run()
 	if err != nil {
-		dataStdErr, _ := ioutil.ReadAll(cmdStderr)
-		dataStdOut, _ := ioutil.ReadAll(cmdStdout)
+		dataStdErr, _ := io.ReadAll(cmdStderr)
+		dataStdOut, _ := io.ReadAll(cmdStdout)
 		cclog.ComponentError(
 			m.name,
 			fmt.Sprintf("Read(): Failed to execute command \"%s\": %v\n", cmd.String(), err),

collectors/infinibandMetric.go

@@ -2,11 +2,10 @@ package collectors
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 	"golang.org/x/sys/unix"
 
 	"encoding/json"
@@ -85,7 +84,7 @@ func (m *InfinibandCollector) Init(config json.RawMessage) error {
 	for _, path := range ibDirs {
 
 		// Skip, when no LID is assigned
-		line, err := ioutil.ReadFile(filepath.Join(path, "lid"))
+		line, err := os.ReadFile(filepath.Join(path, "lid"))
 		if err != nil {
 			continue
 		}
@@ -175,7 +174,7 @@ func (m *InfinibandCollector) Read(interval time.Duration, output chan lp.CCMetr
 		for counterName, counterDef := range info.portCounterFiles {
 
 			// Read counter file
-			line, err := ioutil.ReadFile(counterDef.path)
+			line, err := os.ReadFile(counterDef.path)
 			if err != nil {
 				cclog.ComponentError(
 					m.name,

collectors/iostatMetric.go

@@ -4,8 +4,8 @@ import (
 	"bufio"
 	"os"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 
 	//	"log"
 	"encoding/json"

collectors/ipmiMetric.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"time"
 
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 const IPMITOOL_PATH = `ipmitool`

collectors/likwidMetric.go

@@ -12,7 +12,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math"
 	"os"
 	"os/signal"
@@ -24,10 +23,10 @@ import (
 	"time"
 	"unsafe"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
-	topo "github.com/ClusterCockpit/cc-metric-collector/internal/ccTopology"
 	agg "github.com/ClusterCockpit/cc-metric-collector/internal/metricAggregator"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
+	topo "github.com/ClusterCockpit/cc-metric-collector/pkg/ccTopology"
 	"github.com/NVIDIA/go-nvml/pkg/dl"
 )
 
@@ -154,7 +153,7 @@ func getBaseFreq() float64 {
 	}
 	var freq float64 = math.NaN()
 	for _, f := range files {
-		buffer, err := ioutil.ReadFile(f)
+		buffer, err := os.ReadFile(f)
 		if err == nil {
 			data := strings.Replace(string(buffer), "\n", "", -1)
 			x, err := strconv.ParseInt(data, 0, 64)

collectors/loadavgMetric.go

@@ -3,13 +3,13 @@ package collectors
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"strconv"
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 //
@@ -72,7 +72,7 @@ func (m *LoadavgCollector) Read(interval time.Duration, output chan lp.CCMetric)
 	if !m.init {
 		return
 	}
-	buffer, err := ioutil.ReadFile(LOADAVGFILE)
+	buffer, err := os.ReadFile(LOADAVGFILE)
 	if err != nil {
 		if err != nil {
 			cclog.ComponentError(

collectors/lustreMetric.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 const LUSTRE_SYSFS = `/sys/fs/lustre`

collectors/memstatMetric.go

@@ -12,8 +12,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 const MEMSTATFILE = "/proc/meminfo"
@@ -68,7 +68,8 @@ func getStats(filename string) map[string]MemstatStats {
 		} else if len(linefields) == 5 {
 			v, err := strconv.ParseFloat(linefields[3], 64)
 			if err == nil {
-				stats[strings.Trim(linefields[0], ":")] = MemstatStats{
+				cclog.ComponentDebug("getStats", strings.Trim(linefields[2], ":"), v, linefields[4])
+				stats[strings.Trim(linefields[2], ":")] = MemstatStats{
 					value: v,
 					unit:  linefields[4],
 				}
@@ -160,7 +161,6 @@ func (m *MemstatCollector) Init(config json.RawMessage) error {
 
 func (m *MemstatCollector) Read(interval time.Duration, output chan lp.CCMetric) {
 	if !m.init {
-		cclog.ComponentPrint(m.name, "Here")
 		return
 	}
 
@@ -188,16 +188,20 @@ func (m *MemstatCollector) Read(interval time.Duration, output chan lp.CCMetric)
 			unit := ""
 			if totalVal, total := stats["MemTotal"]; total {
 				if freeVal, free := stats["MemFree"]; free {
+					memUsed = totalVal.value - freeVal.value
+					if len(totalVal.unit) > 0 {
+						unit = totalVal.unit
+					} else if len(freeVal.unit) > 0 {
+						unit = freeVal.unit
+					}
 					if bufVal, buffers := stats["Buffers"]; buffers {
+						memUsed -= bufVal.value
+						if len(bufVal.unit) > 0 && len(unit) == 0 {
+							unit = bufVal.unit
+						}
 						if cacheVal, cached := stats["Cached"]; cached {
-							memUsed = totalVal.value - (freeVal.value + bufVal.value + cacheVal.value)
-							if len(totalVal.unit) > 0 {
-								unit = totalVal.unit
-							} else if len(freeVal.unit) > 0 {
-								unit = freeVal.unit
-							} else if len(bufVal.unit) > 0 {
-								unit = bufVal.unit
-							} else if len(cacheVal.unit) > 0 {
+							memUsed -= cacheVal.value
+							if len(cacheVal.unit) > 0 && len(unit) == 0 {
 								unit = cacheVal.unit
 							}
 						}

collectors/metricCollector.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"time"
 
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 type MetricCollector interface {

collectors/netstatMetric.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 const NETSTATFILE = "/proc/net/dev"

collectors/nfsMetric.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"time"
 
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 // First part contains the code for the general NfsCollector.

collectors/numastatsMetric.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 //

collectors/nvidiaMetric.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 	"github.com/NVIDIA/go-nvml/pkg/nvml"
 )
 

collectors/rocmsmiMetric.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 	"github.com/ClusterCockpit/go-rocm-smi/pkg/rocm_smi"
 )
 
@@ -66,14 +66,14 @@ func (m *RocmSmiCollector) Init(config json.RawMessage) error {
 
 	ret := rocm_smi.Init()
 	if ret != rocm_smi.STATUS_SUCCESS {
-		err = errors.New("Failed to initialize ROCm SMI library")
+		err = errors.New("failed to initialize ROCm SMI library")
 		cclog.ComponentError(m.name, err.Error())
 		return err
 	}
 
 	numDevs, ret := rocm_smi.NumMonitorDevices()
 	if ret != rocm_smi.STATUS_SUCCESS {
-		err = errors.New("Failed to get number of GPUs from ROCm SMI library")
+		err = errors.New("failed to get number of GPUs from ROCm SMI library")
 		cclog.ComponentError(m.name, err.Error())
 		return err
 	}
@@ -98,14 +98,14 @@ func (m *RocmSmiCollector) Init(config json.RawMessage) error {
 		}
 		device, ret := rocm_smi.DeviceGetHandleByIndex(i)
 		if ret != rocm_smi.STATUS_SUCCESS {
-			err = fmt.Errorf("Failed to get handle for GPU %d", i)
+			err = fmt.Errorf("failed to get handle for GPU %d", i)
 			cclog.ComponentError(m.name, err.Error())
 			return err
 		}
 
 		pciInfo, ret := rocm_smi.DeviceGetPciInfo(device)
 		if ret != rocm_smi.STATUS_SUCCESS {
-			err = fmt.Errorf("Failed to get PCI information for GPU %d", i)
+			err = fmt.Errorf("failed to get PCI information for GPU %d", i)
 			cclog.ComponentError(m.name, err.Error())
 			return err
 		}

collectors/sampleMetric.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 // These are the fields we read from the JSON configuration

collectors/sampleTimerMetric.go

@@ -5,8 +5,8 @@ import (
 	"sync"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 // These are the fields we read from the JSON configuration

collectors/schedstatMetric.go

@@ -1,17 +1,17 @@
 package collectors
 
 import (
+	"bufio"
 	"encoding/json"
 	"fmt"
-	"bufio"
-	"time"
-	"os"
-	"strings"
-	"strconv"
 	"math"
+	"os"
+	"strconv"
+	"strings"
+	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 const SCHEDSTATFILE = `/proc/schedstat`
@@ -25,11 +25,11 @@ type SchedstatCollectorConfig struct {
 // defined by metricCollector (name, init, ...)
 type SchedstatCollector struct {
 	metricCollector
-	config SchedstatCollectorConfig // the configuration structure
-	lastTimestamp time.Time                   // Store time stamp of last tick to derive values
-	meta   map[string]string          // default meta information
-	cputags   map[string]map[string]string          // default tags
-	olddata   map[string]map[string]int64          // default tags
+	config        SchedstatCollectorConfig     // the configuration structure
+	lastTimestamp time.Time                    // Store time stamp of last tick to derive values
+	meta          map[string]string            // default meta information
+	cputags       map[string]map[string]string // default tags
+	olddata       map[string]map[string]int64  // default tags
 }
 
 // Functions to implement MetricCollector interface
@@ -52,7 +52,7 @@ func (m *SchedstatCollector) Init(config json.RawMessage) error {
 	// Define meta information sent with each metric
 	// (Can also be dynamic or this is the basic set with extension through AddMeta())
 	m.meta = map[string]string{"source": m.name, "group": "SCHEDSTAT"}
-	
+
 	// Read in the JSON configuration
 	if len(config) > 0 {
 		err = json.Unmarshal(config, &m.config)
@@ -83,12 +83,11 @@ func (m *SchedstatCollector) Init(config json.RawMessage) error {
 			running, _ := strconv.ParseInt(linefields[7], 10, 64)
 			waiting, _ := strconv.ParseInt(linefields[8], 10, 64)
 			m.cputags[linefields[0]] = map[string]string{"type": "hwthread", "type-id": fmt.Sprintf("%d", cpu)}
-			m.olddata[linefields[0]] = map[string]int64{"running" : running, "waiting" : waiting}
+			m.olddata[linefields[0]] = map[string]int64{"running": running, "waiting": waiting}
 			num_cpus++
 		}
 	}
 
-
 	// Save current timestamp
 	m.lastTimestamp = time.Now()
 
@@ -102,7 +101,7 @@ func (m *SchedstatCollector) ParseProcLine(linefields []string, tags map[string]
 	waiting, _ := strconv.ParseInt(linefields[8], 10, 64)
 	diff_running := running - m.olddata[linefields[0]]["running"]
 	diff_waiting := waiting - m.olddata[linefields[0]]["waiting"]
-	
+
 	var l_running float64 = float64(diff_running) / tsdelta.Seconds() / (math.Pow(1000, 3))
 	var l_waiting float64 = float64(diff_waiting) / tsdelta.Seconds() / (math.Pow(1000, 3))
 
@@ -110,11 +109,11 @@ func (m *SchedstatCollector) ParseProcLine(linefields []string, tags map[string]
 	m.olddata[linefields[0]]["waiting"] = waiting
 	value := l_running + l_waiting
 
- 	y, err := lp.New("cpu_load_core", tags, m.meta, map[string]interface{}{"value": value}, now)
+	y, err := lp.New("cpu_load_core", tags, m.meta, map[string]interface{}{"value": value}, now)
 	if err == nil {
 		// Send it to output channel
 		output <- y
-	} 
+	}
 }
 
 // Read collects all metrics belonging to the sample collector

collectors/selfMetric.go

@@ -0,0 +1,144 @@
+package collectors
+
+import (
+	"encoding/json"
+	"runtime"
+	"syscall"
+	"time"
+
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
+)
+
+type SelfCollectorConfig struct {
+	MemStats   bool `json:"read_mem_stats"`
+	GoRoutines bool `json:"read_goroutines"`
+	CgoCalls   bool `json:"read_cgo_calls"`
+	Rusage     bool `json:"read_rusage"`
+}
+
+type SelfCollector struct {
+	metricCollector
+	config SelfCollectorConfig // the configuration structure
+	meta   map[string]string   // default meta information
+	tags   map[string]string   // default tags
+}
+
+func (m *SelfCollector) Init(config json.RawMessage) error {
+	var err error = nil
+	m.name = "SelfCollector"
+	m.setup()
+	m.parallel = true
+	m.meta = map[string]string{"source": m.name, "group": "Self"}
+	m.tags = map[string]string{"type": "node"}
+	if len(config) > 0 {
+		err = json.Unmarshal(config, &m.config)
+		if err != nil {
+			cclog.ComponentError(m.name, "Error reading config:", err.Error())
+			return err
+		}
+	}
+	m.init = true
+	return err
+}
+
+func (m *SelfCollector) Read(interval time.Duration, output chan lp.CCMetric) {
+	timestamp := time.Now()
+
+	if m.config.MemStats {
+		var memstats runtime.MemStats
+		runtime.ReadMemStats(&memstats)
+
+		y, err := lp.New("total_alloc", m.tags, m.meta, map[string]interface{}{"value": memstats.TotalAlloc}, timestamp)
+		if err == nil {
+			y.AddMeta("unit", "Bytes")
+			output <- y
+		}
+		y, err = lp.New("heap_alloc", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapAlloc}, timestamp)
+		if err == nil {
+			y.AddMeta("unit", "Bytes")
+			output <- y
+		}
+		y, err = lp.New("heap_sys", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapSys}, timestamp)
+		if err == nil {
+			y.AddMeta("unit", "Bytes")
+			output <- y
+		}
+		y, err = lp.New("heap_idle", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapIdle}, timestamp)
+		if err == nil {
+			y.AddMeta("unit", "Bytes")
+			output <- y
+		}
+		y, err = lp.New("heap_inuse", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapInuse}, timestamp)
+		if err == nil {
+			y.AddMeta("unit", "Bytes")
+			output <- y
+		}
+		y, err = lp.New("heap_released", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapReleased}, timestamp)
+		if err == nil {
+			y.AddMeta("unit", "Bytes")
+			output <- y
+		}
+		y, err = lp.New("heap_objects", m.tags, m.meta, map[string]interface{}{"value": memstats.HeapObjects}, timestamp)
+		if err == nil {
+			output <- y
+		}
+	}
+	if m.config.GoRoutines {
+		y, err := lp.New("num_goroutines", m.tags, m.meta, map[string]interface{}{"value": runtime.NumGoroutine()}, timestamp)
+		if err == nil {
+			output <- y
+		}
+	}
+	if m.config.CgoCalls {
+		y, err := lp.New("num_cgo_calls", m.tags, m.meta, map[string]interface{}{"value": runtime.NumCgoCall()}, timestamp)
+		if err == nil {
+			output <- y
+		}
+	}
+	if m.config.Rusage {
+		var rusage syscall.Rusage
+		err := syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
+		if err == nil {
+			sec, nsec := rusage.Utime.Unix()
+			t := float64(sec) + (float64(nsec) * 1e-9)
+			y, err := lp.New("rusage_user_time", m.tags, m.meta, map[string]interface{}{"value": t}, timestamp)
+			if err == nil {
+				y.AddMeta("unit", "seconds")
+				output <- y
+			}
+			sec, nsec = rusage.Stime.Unix()
+			t = float64(sec) + (float64(nsec) * 1e-9)
+			y, err = lp.New("rusage_system_time", m.tags, m.meta, map[string]interface{}{"value": t}, timestamp)
+			if err == nil {
+				y.AddMeta("unit", "seconds")
+				output <- y
+			}
+			y, err = lp.New("rusage_vol_ctx_switch", m.tags, m.meta, map[string]interface{}{"value": rusage.Nvcsw}, timestamp)
+			if err == nil {
+				output <- y
+			}
+			y, err = lp.New("rusage_invol_ctx_switch", m.tags, m.meta, map[string]interface{}{"value": rusage.Nivcsw}, timestamp)
+			if err == nil {
+				output <- y
+			}
+			y, err = lp.New("rusage_signals", m.tags, m.meta, map[string]interface{}{"value": rusage.Nsignals}, timestamp)
+			if err == nil {
+				output <- y
+			}
+			y, err = lp.New("rusage_major_pgfaults", m.tags, m.meta, map[string]interface{}{"value": rusage.Majflt}, timestamp)
+			if err == nil {
+				output <- y
+			}
+			y, err = lp.New("rusage_minor_pgfaults", m.tags, m.meta, map[string]interface{}{"value": rusage.Minflt}, timestamp)
+			if err == nil {
+				output <- y
+			}
+		}
+
+	}
+}
+
+func (m *SelfCollector) Close() {
+	m.init = false
+}

collectors/selfMetric.md

@@ -0,0 +1,34 @@
+## `self` collector
+
+```json
+  "self": {
+    "read_mem_stats" : true,
+    "read_goroutines" : true,
+    "read_cgo_calls" : true,
+    "read_rusage" : true
+  }
+```
+
+The `self` collector reads the data from the `runtime` and `syscall` packages, so monitors the execution of the cc-metric-collector itself.
+
+Metrics:
+* If `read_mem_stats == true`:
+  * `total_alloc`: The metric reports cumulative bytes allocated for heap objects.
+  * `heap_alloc`: The metric reports bytes of allocated heap objects.
+  * `heap_sys`: The metric reports bytes of heap memory obtained from the OS.
+  * `heap_idle`: The metric reports bytes in idle (unused) spans.
+  * `heap_inuse`: The metric reports bytes in in-use spans.
+  * `heap_released`: The metric reports bytes of physical memory returned to the OS.
+  * `heap_objects`: The metric reports the number of allocated heap objects.
+* If `read_goroutines == true`:
+  * `num_goroutines`: The metric reports the number of goroutines that currently exist.
+* If `read_cgo_calls == true`:
+  * `num_cgo_calls`: The metric reports the number of cgo calls made by the current process.
+* If `read_rusage == true`:
+  * `rusage_user_time`: The metric reports the amount of time that this process has been scheduled in user mode.
+  * `rusage_system_time`: The metric reports the amount of time that this process has been scheduled in kernel mode.
+  * `rusage_vol_ctx_switch`: The metric reports the amount of voluntary context switches.
+  * `rusage_invol_ctx_switch`: The metric reports the amount of involuntary context switches.
+  * `rusage_signals`: The metric reports the number of signals received.
+  * `rusage_major_pgfaults`: The metric reports the number of major faults the process has made which have required loading a memory page from disk.
+  * `rusage_minor_pgfaults`: The metric reports the number of minor faults the process has made which have not required loading a memory page from disk.

collectors/tempMetric.go

@@ -3,14 +3,14 @@ package collectors
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
-	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	cclog "github.com/ClusterCockpit/cc-metric-collector/pkg/ccLogger"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 // See: https://www.kernel.org/doc/html/latest/hwmon/sysfs-interface.html
@@ -83,14 +83,14 @@ func (m *TempCollector) Init(config json.RawMessage) error {
 
 		// sensor name
 		nameFile := filepath.Join(filepath.Dir(file), "name")
-		name, err := ioutil.ReadFile(nameFile)
+		name, err := os.ReadFile(nameFile)
 		if err == nil {
 			sensor.name = strings.TrimSpace(string(name))
 		}
 
 		// sensor label
 		labelFile := strings.TrimSuffix(file, "_input") + "_label"
-		label, err := ioutil.ReadFile(labelFile)
+		label, err := os.ReadFile(labelFile)
 		if err == nil {
 			sensor.label = strings.TrimSpace(string(label))
 		}
@@ -117,7 +117,7 @@ func (m *TempCollector) Init(config json.RawMessage) error {
 		}
 
 		// Sensor file
-		_, err = ioutil.ReadFile(file)
+		_, err = os.ReadFile(file)
 		if err != nil {
 			continue
 		}
@@ -139,7 +139,7 @@ func (m *TempCollector) Init(config json.RawMessage) error {
 		// max temperature
 		if m.config.ReportMaxTemp {
 			maxTempFile := strings.TrimSuffix(file, "_input") + "_max"
-			if buffer, err := ioutil.ReadFile(maxTempFile); err == nil {
+			if buffer, err := os.ReadFile(maxTempFile); err == nil {
 				if x, err := strconv.ParseInt(strings.TrimSpace(string(buffer)), 10, 64); err == nil {
 					sensor.maxTempName = strings.Replace(sensor.metricName, "temp", "max_temp", 1)
 					sensor.maxTemp = x / 1000
@@ -150,7 +150,7 @@ func (m *TempCollector) Init(config json.RawMessage) error {
 		// critical temperature
 		if m.config.ReportCriticalTemp {
 			criticalTempFile := strings.TrimSuffix(file, "_input") + "_crit"
-			if buffer, err := ioutil.ReadFile(criticalTempFile); err == nil {
+			if buffer, err := os.ReadFile(criticalTempFile); err == nil {
 				if x, err := strconv.ParseInt(strings.TrimSpace(string(buffer)), 10, 64); err == nil {
 					sensor.critTempName = strings.Replace(sensor.metricName, "temp", "crit_temp", 1)
 					sensor.critTemp = x / 1000
@@ -175,7 +175,7 @@ func (m *TempCollector) Read(interval time.Duration, output chan lp.CCMetric) {
 
 	for _, sensor := range m.sensors {
 		// Read sensor file
-		buffer, err := ioutil.ReadFile(sensor.file)
+		buffer, err := os.ReadFile(sensor.file)
 		if err != nil {
 			cclog.ComponentError(
 				m.name,

collectors/topprocsMetric.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	lp "github.com/ClusterCockpit/cc-metric-collector/pkg/ccMetric"
 )
 
 const MAX_NUM_PROCS = 10