Modularize the whole thing (#16)

* Use channels, add a metric router, split up configuration and use extended version of Influx line protocol internally * Use central timer for collectors and router. Add expressions to router * Add expression to router config * Update entry points * Start with README * Update README for CCMetric * Formatting * Update README.md * Add README for MultiChanTicker * Add README for MultiChanTicker * Update README.md * Add README to metric router * Update main README * Remove SinkEntity type * Update README for sinks * Update go files * Update README for receivers * Update collectors README * Update collectors README * Use seperate page per collector * Fix for tempstat page * Add docs for customcmd collector * Add docs for ipmistat collector * Add docs for topprocs collector * Update customCmdMetric.md * Use seconds when calculating LIKWID metrics * Add IB metrics ib_recv_pkts and ib_xmit_pkts * Drop domain part of host name * Updated to latest stable version of likwid * Define source code dependencies in Makefile * Add GPFS / IBM Spectrum Scale collector * Add vet and staticcheck make targets * Add vet and staticcheck make targets * Avoid go vet warning: struct field tag `json:"..., omitempty"` not compatible with reflect.StructTag.Get: suspicious space in struct tag value struct field tag `json:"...", omitempty` not compatible with reflect.StructTag.Get: key:"value" pairs not separated by spaces * Add sample collector to README.md * Add CPU frequency collector * Avoid staticcheck warning: redundant return statement * Avoid staticcheck warning: unnecessary assignment to the blank identifier * Simplified code * Add CPUFreqCollectorCpuinfo a metric collector to measure the current frequency of the CPUs as obtained from /proc/cpuinfo Only measure on the first hyperthread * Add collector for NFS clients * Move publication of metrics into Flush() for NatsSink * Update GitHub actions * Refactoring * Avoid vet warning: Println arg list ends with redundant newline * Avoid vet warning struct field commands has json tag but is not exported * Avoid vet warning: return copies lock value. * Corrected typo * Refactoring * Add go sources in internal/... * Bad separator in Makefile * Fix Infiniband collector Co-authored-by: Holger Obermaier <40787752+ho-ob@users.noreply.github.com>
2025-07-31 08:56:06 +02:00 · 2022-01-25 15:37:43 +01:00
parent 222862af32
commit 200af84c54
60 changed files with 2596 additions and 1105 deletions
--- a/collectors/collectorManager.go
+++ b/collectors/collectorManager.go
@@ -0,0 +1,143 @@
+package collectors
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+	"sync"
+	"time"
+
+	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
+	mct "github.com/ClusterCockpit/cc-metric-collector/internal/multiChanTicker"
+)
+
+var AvailableCollectors = map[string]MetricCollector{
+
+	"likwid":          &LikwidCollector{},
+	"loadavg":         &LoadavgCollector{},
+	"memstat":         &MemstatCollector{},
+	"netstat":         &NetstatCollector{},
+	"ibstat":          &InfinibandCollector{},
+	"lustrestat":      &LustreCollector{},
+	"cpustat":         &CpustatCollector{},
+	"topprocs":        &TopProcsCollector{},
+	"nvidia":          &NvidiaCollector{},
+	"customcmd":       &CustomCmdCollector{},
+	"diskstat":        &DiskstatCollector{},
+	"tempstat":        &TempCollector{},
+	"ipmistat":        &IpmiCollector{},
+	"gpfs":            new(GpfsCollector),
+	"cpufreq":         new(CPUFreqCollector),
+	"cpufreq_cpuinfo": new(CPUFreqCpuInfoCollector),
+	"nfsstat":         new(NfsCollector),
+}
+
+type collectorManager struct {
+	collectors []MetricCollector
+	output     chan lp.CCMetric
+	done       chan bool
+	ticker     mct.MultiChanTicker
+	duration   time.Duration
+	wg         *sync.WaitGroup
+	config     map[string]json.RawMessage
+}
+
+type CollectorManager interface {
+	Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) error
+	AddOutput(output chan lp.CCMetric)
+	Start()
+	Close()
+}
+
+func (cm *collectorManager) Init(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) error {
+	cm.collectors = make([]MetricCollector, 0)
+	cm.output = nil
+	cm.done = make(chan bool)
+	cm.wg = wg
+	cm.ticker = ticker
+	cm.duration = duration
+	configFile, err := os.Open(collectConfigFile)
+	if err != nil {
+		log.Print(err.Error())
+		return err
+	}
+	defer configFile.Close()
+	jsonParser := json.NewDecoder(configFile)
+	err = jsonParser.Decode(&cm.config)
+	if err != nil {
+		log.Print(err.Error())
+		return err
+	}
+	for k, cfg := range cm.config {
+		log.Print(k, " ", cfg)
+		if _, found := AvailableCollectors[k]; !found {
+			log.Print("[CollectorManager] SKIP unknown collector ", k)
+			continue
+		}
+		c := AvailableCollectors[k]
+
+		err = c.Init(cfg)
+		if err != nil {
+			log.Print("[CollectorManager] Collector ", k, "initialization failed: ", err.Error())
+			continue
+		}
+		cm.collectors = append(cm.collectors, c)
+	}
+	return nil
+}
+
+func (cm *collectorManager) Start() {
+	cm.wg.Add(1)
+	tick := make(chan time.Time)
+	cm.ticker.AddChannel(tick)
+	go func() {
+		for {
+		CollectorManagerLoop:
+			select {
+			case <-cm.done:
+				for _, c := range cm.collectors {
+					c.Close()
+				}
+				cm.wg.Done()
+				log.Print("[CollectorManager] DONE\n")
+				break CollectorManagerLoop
+			case t := <-tick:
+				for _, c := range cm.collectors {
+				CollectorManagerInputLoop:
+					select {
+					case <-cm.done:
+						for _, c := range cm.collectors {
+							c.Close()
+						}
+						cm.wg.Done()
+						log.Print("[CollectorManager] DONE\n")
+						break CollectorManagerInputLoop
+					default:
+						log.Print("[CollectorManager] ", c.Name(), " ", t)
+						c.Read(cm.duration, cm.output)
+					}
+				}
+			}
+		}
+		log.Print("[CollectorManager] EXIT\n")
+	}()
+	log.Print("[CollectorManager] STARTED\n")
+}
+
+func (cm *collectorManager) AddOutput(output chan lp.CCMetric) {
+	cm.output = output
+}
+
+func (cm *collectorManager) Close() {
+	cm.done <- true
+	log.Print("[CollectorManager] CLOSE")
+}
+
+func New(ticker mct.MultiChanTicker, duration time.Duration, wg *sync.WaitGroup, collectConfigFile string) (CollectorManager, error) {
+	cm := &collectorManager{}
+	err := cm.Init(ticker, duration, wg, collectConfigFile)
+	if err != nil {
+		return nil, err
+	}
+	return cm, err
+}