Merge branch 'develop' into use_metric_pointers

collectors/likwidMetric.go

@@ -13,41 +13,75 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"math"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
 	"unsafe"
 
+	cclog "github.com/ClusterCockpit/cc-metric-collector/internal/ccLogger"
 	lp "github.com/ClusterCockpit/cc-metric-collector/internal/ccMetric"
-	"gopkg.in/Knetic/govaluate.v2"
+	topo "github.com/ClusterCockpit/cc-metric-collector/internal/ccTopology"
+	mr "github.com/ClusterCockpit/cc-metric-collector/internal/metricRouter"
 )
 
-type MetricScope int
+type MetricScope string
 
 const (
 	METRIC_SCOPE_HWTHREAD = iota
-	METRIC_SCOPE_SOCKET
+	METRIC_SCOPE_CORE
+	METRIC_SCOPE_LLC
 	METRIC_SCOPE_NUMA
+	METRIC_SCOPE_DIE
+	METRIC_SCOPE_SOCKET
 	METRIC_SCOPE_NODE
 )
 
 func (ms MetricScope) String() string {
-	return []string{"Head", "Shoulder", "Knee", "Toe"}[ms]
+	return string(ms)
+}
+
+func (ms MetricScope) Likwid() string {
+	LikwidDomains := map[string]string{
+		"hwthread":   "",
+		"core":       "",
+		"llc":        "C",
+		"numadomain": "M",
+		"die":        "D",
+		"socket":     "S",
+		"node":       "N",
+	}
+	return LikwidDomains[string(ms)]
+}
+
+func (ms MetricScope) Granularity() int {
+	for i, g := range GetAllMetricScopes() {
+		if ms == g {
+			return i
+		}
+	}
+	return -1
+}
+
+func GetAllMetricScopes() []MetricScope {
+	return []MetricScope{"hwthread" /*, "core", "llc", "numadomain", "die",*/, "socket", "node"}
 }
 
 type LikwidCollectorMetricConfig struct {
-	Name    string      `json:"name"`
-	Calc    string      `json:"calc"`
-	Scope   MetricScope `json:"socket_scope"`
-	Publish bool        `json:"publish"`
+	Name        string      `json:"name"`        // Name of the metric
+	Calc        string      `json:"calc"`        // Calculation for the metric using
+	Aggr        string      `json:"aggregation"` // if scope unequal to LIKWID metric scope, the values are combined (sum, min, max, mean or avg, median)
+	Scope       MetricScope `json:"scope"`       // scope for calculation. subscopes are aggregated using the 'aggregation' function
+	Publish     bool        `json:"publish"`
+	granulatity MetricScope
 }
 
 type LikwidCollectorEventsetConfig struct {
-	Events  map[string]string             `json:"events"`
-	Metrics []LikwidCollectorMetricConfig `json:"metrics"`
+	Events      map[string]string `json:"events"`
+	granulatity map[string]MetricScope
+	Metrics     []LikwidCollectorMetricConfig `json:"metrics"`
 }
 
 type LikwidCollectorConfig struct {
@@ -59,22 +93,25 @@ type LikwidCollectorConfig struct {
 
 type LikwidCollector struct {
 	metricCollector
-	cpulist   []C.int
-	sock2tid  map[int]int
-	metrics   map[C.int]map[string]int
-	groups    []C.int
-	config    LikwidCollectorConfig
-	results   map[int]map[int]map[string]interface{}
-	mresults  map[int]map[int]map[string]float64
-	gmresults map[int]map[string]float64
-	basefreq  float64
+	cpulist       []C.int
+	cpu2tid       map[int]int
+	sock2tid      map[int]int
+	scopeRespTids map[MetricScope]map[int]int
+	metrics       map[C.int]map[string]int
+	groups        []C.int
+	config        LikwidCollectorConfig
+	results       map[int]map[int]map[string]interface{}
+	mresults      map[int]map[int]map[string]float64
+	gmresults     map[int]map[string]float64
+	basefreq      float64
+	running       bool
 }
 
 type LikwidMetric struct {
-	name         string
-	search       string
-	socket_scope bool
-	group_idx    int
+	name      string
+	search    string
+	scope     MetricScope
+	group_idx int
 }
 
 func eventsToEventStr(events map[string]string) string {
@@ -85,6 +122,21 @@ func eventsToEventStr(events map[string]string) string {
 	return strings.Join(elist, ",")
 }
 
+func getGranularity(counter, event string) MetricScope {
+	if strings.HasPrefix(counter, "PMC") || strings.HasPrefix(counter, "FIXC") {
+		return "hwthread"
+	} else if strings.Contains(counter, "BOX") || strings.Contains(counter, "DEV") {
+		return "socket"
+	} else if strings.HasPrefix(counter, "PWR") {
+		if event == "RAPL_CORE_ENERGY" {
+			return "hwthread"
+		} else {
+			return "socket"
+		}
+	}
+	return "unknown"
+}
+
 func getBaseFreq() float64 {
 	var freq float64 = math.NaN()
 	C.power_init(0)
@@ -104,18 +156,98 @@ func getBaseFreq() float64 {
 	return freq
 }
 
-func getSocketCpus() map[C.int]int {
-	slist := SocketList()
-	var cpu C.int
-	outmap := make(map[C.int]int)
-	for _, s := range slist {
-		t := C.CString(fmt.Sprintf("S%d", s))
-		clen := C.cpustr_to_cpulist(t, &cpu, 1)
-		if int(clen) == 1 {
-			outmap[cpu] = s
+func (m *LikwidCollector) initGranularity() {
+	splitRegex := regexp.MustCompile("[+-/*()]")
+	for _, evset := range m.config.Eventsets {
+		evset.granulatity = make(map[string]MetricScope)
+		for counter, event := range evset.Events {
+			gran := getGranularity(counter, event)
+			if gran.Granularity() >= 0 {
+				evset.granulatity[counter] = gran
+			}
+		}
+		for i, metric := range evset.Metrics {
+			s := splitRegex.Split(metric.Calc, -1)
+			gran := MetricScope("hwthread")
+			evset.Metrics[i].granulatity = gran
+			for _, x := range s {
+				if _, ok := evset.Events[x]; ok {
+					if evset.granulatity[x].Granularity() > gran.Granularity() {
+						gran = evset.granulatity[x]
+					}
+				}
+			}
+			evset.Metrics[i].granulatity = gran
 		}
 	}
-	return outmap
+	for i, metric := range m.config.Metrics {
+		s := splitRegex.Split(metric.Calc, -1)
+		gran := MetricScope("hwthread")
+		m.config.Metrics[i].granulatity = gran
+		for _, x := range s {
+			for _, evset := range m.config.Eventsets {
+				for _, m := range evset.Metrics {
+					if m.Name == x && m.granulatity.Granularity() > gran.Granularity() {
+						gran = m.granulatity
+					}
+				}
+			}
+		}
+		m.config.Metrics[i].granulatity = gran
+	}
+}
+
+type TopoResolveFunc func(cpuid int) int
+
+func (m *LikwidCollector) getResponsiblities() map[MetricScope]map[int]int {
+	get_cpus := func(scope MetricScope) map[int]int {
+		var slist []int
+		var cpu C.int
+		var input func(index int) string
+		switch scope {
+		case "node":
+			slist = []int{0}
+			input = func(index int) string { return "N:0" }
+		case "socket":
+			input = func(index int) string { return fmt.Sprintf("%s%d:0", scope.Likwid(), index) }
+			slist = topo.SocketList()
+		// case "numadomain":
+		// 	input = func(index int) string { return fmt.Sprintf("%s%d:0", scope.Likwid(), index) }
+		// 	slist = topo.NumaNodeList()
+		// 	cclog.Debug(scope, " ", input(0), " ", slist)
+		// case "die":
+		// 	input = func(index int) string { return fmt.Sprintf("%s%d:0", scope.Likwid(), index) }
+		// 	slist = topo.DieList()
+		// case "llc":
+		// 	input = fmt.Sprintf("%s%d:0", scope.Likwid(), s)
+		// 	slist = topo.LLCacheList()
+		case "hwthread":
+			input = func(index int) string { return fmt.Sprintf("%d", index) }
+			slist = topo.CpuList()
+		}
+		outmap := make(map[int]int)
+		for _, s := range slist {
+			t := C.CString(input(s))
+			clen := C.cpustr_to_cpulist(t, &cpu, 1)
+			if int(clen) == 1 {
+				outmap[s] = m.cpu2tid[int(cpu)]
+			} else {
+				cclog.Error(fmt.Sprintf("Cannot determine responsible CPU for %s", input(s)))
+				outmap[s] = -1
+			}
+			C.free(unsafe.Pointer(t))
+		}
+		return outmap
+	}
+
+	scopes := GetAllMetricScopes()
+	complete := make(map[MetricScope]map[int]int)
+	for _, s := range scopes {
+		cclog.Debug("Start ", s)
+		complete[s] = get_cpus(s)
+		cclog.Debug("End ", s)
+	}
+	return complete
 }
 
 func (m *LikwidCollector) Init(config json.RawMessage) error {
@@ -127,38 +259,74 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 			return err
 		}
 	}
+	if m.config.ForceOverwrite {
+		cclog.ComponentDebug(m.name, "Set LIKWID_FORCE=1")
+		os.Setenv("LIKWID_FORCE", "1")
+	}
 	m.setup()
-	m.meta = map[string]string{"source": m.name, "group": "PerfCounter"}
-	cpulist := CpuList()
-	m.cpulist = make([]C.int, len(cpulist))
-	slist := getSocketCpus()
 
-	m.sock2tid = make(map[int]int)
-	//	m.numa2tid = make(map[int]int)
+	m.meta = map[string]string{"source": m.name, "group": "PerfCounter"}
+	cclog.ComponentDebug(m.name, "Get cpulist and init maps and lists")
+	cpulist := topo.CpuList()
+	m.cpulist = make([]C.int, len(cpulist))
+	m.cpu2tid = make(map[int]int)
 	for i, c := range cpulist {
 		m.cpulist[i] = C.int(c)
-		if sid, found := slist[m.cpulist[i]]; found {
-			m.sock2tid[sid] = i
-		}
+		m.cpu2tid[c] = i
+
 	}
 	m.results = make(map[int]map[int]map[string]interface{})
 	m.mresults = make(map[int]map[int]map[string]float64)
 	m.gmresults = make(map[int]map[string]float64)
+	cclog.ComponentDebug(m.name, "initialize LIKWID topology")
 	ret = C.topology_init()
 	if ret != 0 {
-		return errors.New("Failed to initialize LIKWID topology")
-	}
-	if m.config.ForceOverwrite {
-		os.Setenv("LIKWID_FORCE", "1")
+		err := errors.New("failed to initialize LIKWID topology")
+		cclog.ComponentError(m.name, err.Error())
+		return err
 	}
+
+	// Determine which counter works at which level. PMC*: hwthread, *BOX*: socket, ...
+	m.initGranularity()
+	// Generate map for MetricScope -> scope_id (like socket id) -> responsible id (offset in cpulist)
+	m.scopeRespTids = m.getResponsiblities()
+
+	cclog.ComponentDebug(m.name, "initialize LIKWID perfmon module")
 	ret = C.perfmon_init(C.int(len(m.cpulist)), &m.cpulist[0])
 	if ret != 0 {
 		C.topology_finalize()
-		return errors.New("Failed to initialize LIKWID topology")
+		err := errors.New("failed to initialize LIKWID topology")
+		cclog.ComponentError(m.name, err.Error())
+		return err
 	}
 
+	// This is for the global metrics computation test
+	globalParams := make(map[string]interface{})
+	globalParams["time"] = float64(1.0)
+	globalParams["inverseClock"] = float64(1.0)
+	// While adding the events, we test the metrics whether they can be computed at all
 	for i, evset := range m.config.Eventsets {
 		estr := eventsToEventStr(evset.Events)
+		// Generate parameter list for the metric computing test
+		params := make(map[string]interface{})
+		params["time"] = float64(1.0)
+		params["inverseClock"] = float64(1.0)
+		for counter := range evset.Events {
+			params[counter] = float64(1.0)
+		}
+		for _, metric := range evset.Metrics {
+			// Try to evaluate the metric
+			_, err := mr.EvalFloat64Condition(metric.Calc, params)
+			if err != nil {
+				cclog.ComponentError(m.name, "Calculation for metric", metric.Name, "failed:", err.Error())
+				continue
+			}
+			// If the metric is not in the parameter list for the global metrics, add it
+			if _, ok := globalParams[metric.Name]; !ok {
+				globalParams[metric.Name] = float64(1.0)
+			}
+		}
+		// Now we add the list of events to likwid
 		cstr := C.CString(estr)
 		gid := C.perfmon_addEventSet(cstr)
 		if gid >= 0 {
@@ -170,161 +338,191 @@ func (m *LikwidCollector) Init(config json.RawMessage) error {
 		for tid := range m.cpulist {
 			m.results[i][tid] = make(map[string]interface{})
 			m.mresults[i][tid] = make(map[string]float64)
-			m.gmresults[tid] = make(map[string]float64)
+			if i == 0 {
+				m.gmresults[tid] = make(map[string]float64)
+			}
+		}
+	}
+	for _, metric := range m.config.Metrics {
+		// Try to evaluate the global metric
+		_, err := mr.EvalFloat64Condition(metric.Calc, globalParams)
+		if err != nil {
+			cclog.ComponentError(m.name, "Calculation for metric", metric.Name, "failed:", err.Error())
+			continue
 		}
 	}
 
+	// If no event set could be added, shut down LikwidCollector
 	if len(m.groups) == 0 {
 		C.perfmon_finalize()
 		C.topology_finalize()
-		return errors.New("No LIKWID performance group initialized")
+		err := errors.New("no LIKWID performance group initialized")
+		cclog.ComponentError(m.name, err.Error())
+		return err
 	}
 	m.basefreq = getBaseFreq()
 	m.init = true
 	return nil
 }
 
-func (m *LikwidCollector) Read(interval time.Duration, output chan *lp.CCMetric) {
+
+// take a measurement for 'interval' seconds of event set index 'group'
+func (m *LikwidCollector) takeMeasurement(group int, interval time.Duration) error {
+
+	var ret C.int
+	gid := m.groups[group]
+	ret = C.perfmon_setupCounters(gid)
+	if ret != 0 {
+		gctr := C.GoString(C.perfmon_getGroupName(gid))
+		err := fmt.Errorf("failed to setup performance group %s", gctr)
+		cclog.ComponentError(m.name, err.Error())
+		return err
+	}
+	ret = C.perfmon_startCounters()
+	if ret != 0 {
+		gctr := C.GoString(C.perfmon_getGroupName(gid))
+		err := fmt.Errorf("failed to start performance group %s", gctr)
+		cclog.ComponentError(m.name, err.Error())
+		return err
+	}
+	m.running = true
+	time.Sleep(interval)
+	m.running = false
+	ret = C.perfmon_stopCounters()
+	if ret != 0 {
+		gctr := C.GoString(C.perfmon_getGroupName(gid))
+		err := fmt.Errorf("failed to stop performance group %s", gctr)
+		cclog.ComponentError(m.name, err.Error())
+		return err
+	}
+	return nil
+}
+
+// Get all measurement results for an event set, derive the metric values out of the measurement results and send it
+func (m *LikwidCollector) calcEventsetMetrics(group int, interval time.Duration, output chan lp.CCMetric) error {
+	var eidx C.int
+	evset := m.config.Eventsets[group]
+	gid := m.groups[group]
+
+	// Go over events and get the results
+	for eidx = 0; int(eidx) < len(evset.Events); eidx++ {
+		ctr := C.perfmon_getCounterName(gid, eidx)
+		ev := C.perfmon_getEventName(gid, eidx)
+		gctr := C.GoString(ctr)
+		gev := C.GoString(ev)
+		// MetricScope for the counter (and if needed the event)
+		scope := getGranularity(gctr, gev)
+		// Get the map scope-id -> tids
+		// This way we read less counters like only the responsible hardware thread for a socket
+		scopemap := m.scopeRespTids[scope]
+		for _, tid := range scopemap {
+			if tid >= 0 {
+				m.results[group][tid]["time"] = interval.Seconds()
+				m.results[group][tid]["inverseClock"] = float64(1.0 / m.basefreq)
+				res := C.perfmon_getLastResult(gid, eidx, C.int(tid))
+				m.results[group][tid][gctr] = float64(res)
+			}
+		}
+	}
+
+	// Go over the event set metrics, derive the value out of the event:counter values and send it
+	for _, metric := range evset.Metrics {
+		// The metric scope is determined in the Init() function
+		// Get the map scope-id -> tids
+		scopemap := m.scopeRespTids[metric.Scope]
+		for domain, tid := range scopemap {
+			if tid >= 0 {
+				value, err := mr.EvalFloat64Condition(metric.Calc, m.results[group][tid])
+				if err != nil {
+					cclog.ComponentError(m.name, "Calculation for metric", metric.Name, "failed:", err.Error())
+					continue
+				}
+				m.mresults[group][tid][metric.Name] = value
+				// Now we have the result, send it with the proper tags
+				tags := map[string]string{"type": metric.Scope.String()}
+				if metric.Scope != "node" {
+					tags["type-id"] = fmt.Sprintf("%d", domain)
+				}
+				fields := map[string]interface{}{"value": value}
+				y, err := lp.New(metric.Name, tags, m.meta, fields, time.Now())
+				if err == nil {
+					output <- y
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// Go over the global metrics, derive the value out of the event sets' metric values and send it
+func (m *LikwidCollector) calcGlobalMetrics(interval time.Duration, output chan lp.CCMetric) error {
+	for _, metric := range m.config.Metrics {
+		scopemap := m.scopeRespTids[metric.Scope]
+		for domain, tid := range scopemap {
+			if tid >= 0 {
+				// Here we generate parameter list
+				params := make(map[string]interface{})
+				for j := range m.groups {
+					for mname, mres := range m.mresults[j][tid] {
+						params[mname] = mres
+					}
+				}
+				// Evaluate the metric
+				value, err := mr.EvalFloat64Condition(metric.Calc, params)
+				if err != nil {
+					cclog.ComponentError(m.name, "Calculation for metric", metric.Name, "failed:", err.Error())
+					continue
+				}
+				m.gmresults[tid][metric.Name] = value
+				// Now we have the result, send it with the proper tags
+				tags := map[string]string{"type": metric.Scope.String()}
+				if metric.Scope != "node" {
+					tags["type-id"] = fmt.Sprintf("%d", domain)
+				}
+				fields := map[string]interface{}{"value": value}
+				y, err := lp.New(metric.Name, tags, m.meta, fields, time.Now())
+				if err == nil {
+					output <- y
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// main read function taking multiple measurement rounds, each 'interval' seconds long
+func (m *LikwidCollector) Read(interval time.Duration, output chan lp.CCMetric) {
 	if !m.init {
 		return
 	}
-	var ret C.int
 
-	for i, gid := range m.groups {
-		evset := m.config.Eventsets[i]
-		ret = C.perfmon_setupCounters(gid)
-		if ret != 0 {
-			log.Print("Failed to setup performance group ", C.perfmon_getGroupName(gid))
+	for i, _ := range m.groups {
+		// measure event set 'i' for 'interval' seconds
+		err := m.takeMeasurement(i, interval)
+		if err != nil {
+			cclog.ComponentError(m.name, err.Error())
 			continue
 		}
-		ret = C.perfmon_startCounters()
-		if ret != 0 {
-			log.Print("Failed to start performance group ", C.perfmon_getGroupName(gid))
-			continue
-		}
-		time.Sleep(interval)
-		ret = C.perfmon_stopCounters()
-		if ret != 0 {
-			log.Print("Failed to stop performance group ", C.perfmon_getGroupName(gid))
-			continue
-		}
-		var eidx C.int
-		for tid := range m.cpulist {
-			for eidx = 0; int(eidx) < len(evset.Events); eidx++ {
-				ctr := C.perfmon_getCounterName(gid, eidx)
-				gctr := C.GoString(ctr)
-				res := C.perfmon_getLastResult(gid, eidx, C.int(tid))
-				m.results[i][tid][gctr] = float64(res)
-			}
-			m.results[i][tid]["time"] = interval.Seconds()
-			m.results[i][tid]["inverseClock"] = float64(1.0 / m.basefreq)
-			for _, metric := range evset.Metrics {
-				expression, err := govaluate.NewEvaluableExpression(metric.Calc)
-				if err != nil {
-					log.Print(err.Error())
-					continue
-				}
-				result, err := expression.Evaluate(m.results[i][tid])
-				if err != nil {
-					log.Print(err.Error())
-					continue
-				}
-				m.mresults[i][tid][metric.Name] = float64(result.(float64))
-			}
-		}
-	}
-
-	for _, metric := range m.config.Metrics {
-		for tid := range m.cpulist {
-			var params map[string]interface{}
-			expression, err := govaluate.NewEvaluableExpression(metric.Calc)
-			if err != nil {
-				log.Print(err.Error())
-				continue
-			}
-			params = make(map[string]interface{})
-			for j := range m.groups {
-				for mname, mres := range m.mresults[j][tid] {
-					params[mname] = mres
-				}
-			}
-			result, err := expression.Evaluate(params)
-			if err != nil {
-				log.Print(err.Error())
-				continue
-			}
-			m.gmresults[tid][metric.Name] = float64(result.(float64))
-		}
-	}
-	for i := range m.groups {
-		evset := m.config.Eventsets[i]
-		for _, metric := range evset.Metrics {
-			_, skip := stringArrayContains(m.config.ExcludeMetrics, metric.Name)
-			if metric.Publish && !skip {
-				if metric.Scope.String() == "socket" {
-					for sid, tid := range m.sock2tid {
-						y, err := lp.New(metric.Name,
-							map[string]string{"type": "socket",
-								"type-id": fmt.Sprintf("%d", int(sid))},
-							m.meta,
-							map[string]interface{}{"value": m.mresults[i][tid][metric.Name]},
-							time.Now())
-						if err == nil {
-							output <- &y
-						}
-					}
-				} else if metric.Scope.String() == "hwthread" {
-					for tid, cpu := range m.cpulist {
-						y, err := lp.New(metric.Name,
-							map[string]string{"type": "cpu",
-								"type-id": fmt.Sprintf("%d", int(cpu))},
-							m.meta,
-							map[string]interface{}{"value": m.mresults[i][tid][metric.Name]},
-							time.Now())
-						if err == nil {
-							output <- &y
-						}
-					}
-				}
-			}
-		}
-	}
-	for _, metric := range m.config.Metrics {
-		_, skip := stringArrayContains(m.config.ExcludeMetrics, metric.Name)
-		if metric.Publish && !skip {
-			if metric.Scope.String() == "socket" {
-				for sid, tid := range m.sock2tid {
-					y, err := lp.New(metric.Name,
-						map[string]string{"type": "socket",
-							"type-id": fmt.Sprintf("%d", int(sid))},
-						m.meta,
-						map[string]interface{}{"value": m.gmresults[tid][metric.Name]},
-						time.Now())
-					if err == nil {
-						output <- &y
-					}
-				}
-			} else {
-				for tid, cpu := range m.cpulist {
-					y, err := lp.New(metric.Name,
-						map[string]string{"type": "cpu",
-							"type-id": fmt.Sprintf("%d", int(cpu))},
-						m.meta,
-						map[string]interface{}{"value": m.gmresults[tid][metric.Name]},
-						time.Now())
-					if err == nil {
-						output <- &y
-					}
-				}
-			}
-		}
+		// read measurements and derive event set metrics
+		m.calcEventsetMetrics(i, interval, output)
 	}
+	// use the event set metrics to derive the global metrics
+	m.calcGlobalMetrics(interval, output)
 }
 
 func (m *LikwidCollector) Close() {
 	if m.init {
+		cclog.ComponentDebug(m.name, "Closing ...")
 		m.init = false
+		if m.running {
+			cclog.ComponentDebug(m.name, "Stopping counters")
+			C.perfmon_stopCounters()
+		}
+		cclog.ComponentDebug(m.name, "Finalize LIKWID perfmon module")
 		C.perfmon_finalize()
+		cclog.ComponentDebug(m.name, "Finalize LIKWID topology module")
 		C.topology_finalize()
+		cclog.ComponentDebug(m.name, "Closing done")
 	}
 }