cc-backend/pkg/schema/metrics.go

318 lines
6.8 KiB
Go
Raw Normal View History

// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
2021-10-26 10:22:02 +02:00
package schema
import (
"fmt"
"io"
2022-01-20 10:08:50 +01:00
"math"
"sort"
"unsafe"
2021-10-26 10:22:02 +02:00
)
2022-01-07 09:39:00 +01:00
type JobData map[string]map[MetricScope]*JobMetric
2021-10-26 10:22:02 +02:00
type JobMetric struct {
2022-10-04 10:12:35 +02:00
Unit Unit `json:"unit"`
2022-01-07 09:39:00 +01:00
Timestep int `json:"timestep"`
Series []Series `json:"series"`
StatisticsSeries *StatsSeries `json:"statisticsSeries,omitempty"`
2021-12-17 15:49:22 +01:00
}
type Series struct {
Hostname string `json:"hostname"`
Id *string `json:"id,omitempty"`
Statistics MetricStatistics `json:"statistics"`
Data []Float `json:"data"`
2021-12-17 15:49:22 +01:00
}
type MetricStatistics struct {
Avg float64 `json:"avg"`
Min float64 `json:"min"`
Max float64 `json:"max"`
}
type StatsSeries struct {
2022-01-07 09:39:00 +01:00
Mean []Float `json:"mean"`
Min []Float `json:"min"`
Max []Float `json:"max"`
2021-12-17 15:49:22 +01:00
Percentiles map[int][]Float `json:"percentiles,omitempty"`
2021-10-26 10:22:02 +02:00
}
type MetricScope string
const (
2022-01-20 10:08:50 +01:00
MetricScopeInvalid MetricScope = "invalid_scope"
MetricScopeNode MetricScope = "node"
MetricScopeSocket MetricScope = "socket"
MetricScopeMemoryDomain MetricScope = "memoryDomain"
MetricScopeCore MetricScope = "core"
MetricScopeHWThread MetricScope = "hwthread"
2022-01-12 13:03:01 +01:00
MetricScopeAccelerator MetricScope = "accelerator"
2021-10-26 10:22:02 +02:00
)
2022-01-07 09:39:00 +01:00
var metricScopeGranularity map[MetricScope]int = map[MetricScope]int{
MetricScopeNode: 10,
MetricScopeSocket: 5,
MetricScopeMemoryDomain: 3,
MetricScopeCore: 2,
MetricScopeHWThread: 1,
2022-01-12 13:03:01 +01:00
MetricScopeAccelerator: 5, // Special/Randomly choosen
2022-01-20 10:08:50 +01:00
MetricScopeInvalid: -1,
2022-01-07 09:39:00 +01:00
}
func (e *MetricScope) LT(other MetricScope) bool {
2022-01-07 09:39:00 +01:00
a := metricScopeGranularity[*e]
b := metricScopeGranularity[other]
2022-01-12 13:03:01 +01:00
return a < b
2022-01-07 09:39:00 +01:00
}
func (e *MetricScope) LTE(other MetricScope) bool {
a := metricScopeGranularity[*e]
b := metricScopeGranularity[other]
return a <= b
}
func (e *MetricScope) Max(other MetricScope) MetricScope {
a := metricScopeGranularity[*e]
b := metricScopeGranularity[other]
if a > b {
return *e
}
return other
}
2021-10-26 10:22:02 +02:00
func (e *MetricScope) UnmarshalGQL(v interface{}) error {
str, ok := v.(string)
if !ok {
return fmt.Errorf("SCHEMA/METRICS > enums must be strings")
2021-10-26 10:22:02 +02:00
}
*e = MetricScope(str)
2022-03-01 16:00:44 +01:00
if !e.Valid() {
return fmt.Errorf("SCHEMA/METRICS > %s is not a valid MetricScope", str)
2021-10-26 10:22:02 +02:00
}
return nil
}
func (e MetricScope) MarshalGQL(w io.Writer) {
fmt.Fprintf(w, "\"%s\"", e)
}
2022-03-01 16:00:44 +01:00
func (e MetricScope) Valid() bool {
gran, ok := metricScopeGranularity[e]
return ok && gran > 0
}
func (jd *JobData) Size() int {
n := 128
for _, scopes := range *jd {
for _, metric := range scopes {
if metric.StatisticsSeries != nil {
n += len(metric.StatisticsSeries.Max)
n += len(metric.StatisticsSeries.Mean)
n += len(metric.StatisticsSeries.Min)
}
for _, series := range metric.Series {
n += len(series.Data)
}
}
}
return n * int(unsafe.Sizeof(Float(0)))
}
2022-01-20 10:08:50 +01:00
const smooth bool = false
func (jm *JobMetric) AddStatisticsSeries() {
if jm.StatisticsSeries != nil || len(jm.Series) < 4 {
return
}
n, m := 0, len(jm.Series[0].Data)
for _, series := range jm.Series {
if len(series.Data) > n {
n = len(series.Data)
}
if len(series.Data) < m {
m = len(series.Data)
}
}
min, mean, max := make([]Float, n), make([]Float, n), make([]Float, n)
i := 0
for ; i < m; i++ {
smin, ssum, smax := math.MaxFloat32, 0.0, -math.MaxFloat32
notnan := 0
for j := 0; j < len(jm.Series); j++ {
x := float64(jm.Series[j].Data[i])
if math.IsNaN(x) {
continue
}
notnan += 1
ssum += x
smin = math.Min(smin, x)
smax = math.Max(smax, x)
}
if notnan < 3 {
min[i] = NaN
mean[i] = NaN
max[i] = NaN
} else {
min[i] = Float(smin)
mean[i] = Float(ssum / float64(notnan))
max[i] = Float(smax)
}
}
for ; i < n; i++ {
min[i] = NaN
mean[i] = NaN
max[i] = NaN
}
if smooth {
for i := 2; i < len(mean)-2; i++ {
if min[i].IsNaN() {
continue
}
min[i] = (min[i-2] + min[i-1] + min[i] + min[i+1] + min[i+2]) / 5
max[i] = (max[i-2] + max[i-1] + max[i] + max[i+1] + max[i+2]) / 5
mean[i] = (mean[i-2] + mean[i-1] + mean[i] + mean[i+1] + mean[i+2]) / 5
}
}
jm.StatisticsSeries = &StatsSeries{Mean: mean, Min: min, Max: max}
}
func (jd *JobData) AddNodeScope(metric string) bool {
scopes, ok := (*jd)[metric]
if !ok {
return false
}
var maxScope MetricScope = MetricScopeInvalid
for scope := range scopes {
maxScope = maxScope.Max(scope)
}
if maxScope == MetricScopeInvalid || maxScope == MetricScopeNode {
return false
}
jm := scopes[maxScope]
hosts := make(map[string][]Series, 32)
for _, series := range jm.Series {
hosts[series.Hostname] = append(hosts[series.Hostname], series)
}
nodeJm := &JobMetric{
Unit: jm.Unit,
Timestep: jm.Timestep,
Series: make([]Series, 0, len(hosts)),
}
for hostname, series := range hosts {
min, sum, max := math.MaxFloat32, 0.0, -math.MaxFloat32
for _, series := range series {
sum += series.Statistics.Avg
min = math.Min(min, series.Statistics.Min)
max = math.Max(max, series.Statistics.Max)
}
n, m := 0, len(jm.Series[0].Data)
for _, series := range jm.Series {
if len(series.Data) > n {
n = len(series.Data)
}
if len(series.Data) < m {
m = len(series.Data)
}
}
i, data := 0, make([]Float, len(series[0].Data))
for ; i < m; i++ {
x := Float(0.0)
for _, series := range jm.Series {
x += series.Data[i]
}
data[i] = x
}
for ; i < n; i++ {
data[i] = NaN
}
nodeJm.Series = append(nodeJm.Series, Series{
Hostname: hostname,
Statistics: MetricStatistics{Min: min, Avg: sum / float64(len(series)), Max: max},
2022-01-20 10:08:50 +01:00
Data: data,
})
}
scopes[MetricScopeNode] = nodeJm
return true
}
func (jm *JobMetric) AddPercentiles(ps []int) bool {
if jm.StatisticsSeries == nil {
jm.AddStatisticsSeries()
}
if len(jm.Series) < 3 {
return false
}
if jm.StatisticsSeries.Percentiles == nil {
jm.StatisticsSeries.Percentiles = make(map[int][]Float, len(ps))
}
n := 0
for _, series := range jm.Series {
if len(series.Data) > n {
n = len(series.Data)
}
}
data := make([][]float64, n)
for i := 0; i < n; i++ {
vals := make([]float64, 0, len(jm.Series))
for _, series := range jm.Series {
if i < len(series.Data) {
vals = append(vals, float64(series.Data[i]))
}
}
sort.Float64s(vals)
data[i] = vals
}
for _, p := range ps {
if p < 1 || p > 99 {
panic("SCHEMA/METRICS > invalid percentile")
2022-01-20 10:08:50 +01:00
}
if _, ok := jm.StatisticsSeries.Percentiles[p]; ok {
continue
}
percentiles := make([]Float, n)
for i := 0; i < n; i++ {
sorted := data[i]
percentiles[i] = Float(sorted[(len(sorted)*p)/100])
}
jm.StatisticsSeries.Percentiles[p] = percentiles
}
return true
}