cc-backend/internal/metricdata/metricdata.go

357 lines
9.7 KiB
Go
Raw Normal View History

2024-04-11 23:04:30 +02:00
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package metricdata
import (
"context"
"encoding/json"
2021-11-26 10:32:36 +01:00
"fmt"
"math"
"time"
2022-06-21 17:52:36 +02:00
"github.com/ClusterCockpit/cc-backend/internal/config"
"github.com/ClusterCockpit/cc-backend/pkg/archive"
2022-06-21 17:52:36 +02:00
"github.com/ClusterCockpit/cc-backend/pkg/log"
2022-06-22 06:11:00 +02:00
"github.com/ClusterCockpit/cc-backend/pkg/lrucache"
2022-06-21 17:52:36 +02:00
"github.com/ClusterCockpit/cc-backend/pkg/schema"
)
2021-12-08 10:14:45 +01:00
type MetricDataRepository interface {
2021-12-09 16:25:48 +01:00
// Initialize this MetricDataRepository. One instance of
// this interface will only ever be responsible for one cluster.
Init(rawConfig json.RawMessage) error
2021-12-09 16:25:48 +01:00
// Return the JobData for the given job, only with the requested metrics.
LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error)
2021-12-09 16:25:48 +01:00
2022-01-12 13:03:01 +01:00
// Return a map of metrics to a map of nodes to the metric statistics of the job. node scope assumed for now.
2021-12-17 15:49:22 +01:00
LoadStats(job *schema.Job, metrics []string, ctx context.Context) (map[string]map[string]schema.MetricStatistics, error)
2021-12-09 16:25:48 +01:00
// Return a map of hosts to a map of metrics at the requested scopes for that node.
LoadNodeData(cluster string, metrics, nodes []string, scopes []schema.MetricScope, from, to time.Time, ctx context.Context) (map[string]map[string][]*schema.JobMetric, error)
2021-12-08 10:14:45 +01:00
}
var metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{}
2021-11-26 10:32:36 +01:00
var useArchive bool
func Init(disableArchive bool) error {
useArchive = !disableArchive
for _, cluster := range config.Keys.Clusters {
2021-12-08 10:14:45 +01:00
if cluster.MetricDataRepository != nil {
var kind struct {
Kind string `json:"kind"`
}
if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil {
log.Warn("Error while unmarshaling raw json MetricDataRepository")
return err
}
2022-01-24 10:06:25 +01:00
var mdr MetricDataRepository
switch kind.Kind {
2021-12-08 10:14:45 +01:00
case "cc-metric-store":
2022-01-24 10:06:25 +01:00
mdr = &CCMetricStore{}
case "influxdb":
mdr = &InfluxDBv2DataRepository{}
2022-12-08 13:51:44 +01:00
case "prometheus":
mdr = &PrometheusDataRepository{}
2022-01-24 10:06:25 +01:00
case "test":
mdr = &TestMetricDataRepository{}
2021-12-08 10:14:45 +01:00
default:
return fmt.Errorf("METRICDATA/METRICDATA > Unknown MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name)
2021-12-08 10:14:45 +01:00
}
2022-01-24 10:06:25 +01:00
if err := mdr.Init(cluster.MetricDataRepository); err != nil {
log.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name)
2022-01-24 10:06:25 +01:00
return err
}
metricDataRepos[cluster.Name] = mdr
2021-12-08 10:14:45 +01:00
}
2021-11-26 10:32:36 +01:00
}
2021-12-08 10:14:45 +01:00
return nil
2021-11-26 10:32:36 +01:00
}
2022-04-07 12:28:09 +02:00
var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024)
// Fetches the metric data for a job.
func LoadData(job *schema.Job,
metrics []string,
scopes []schema.MetricScope,
2024-02-23 10:08:41 +01:00
ctx context.Context,
) (schema.JobData, error) {
2022-09-13 15:21:50 +02:00
data := cache.Get(cacheKey(job, metrics, scopes), func() (_ interface{}, ttl time.Duration, size int) {
var jd schema.JobData
var err error
2022-09-13 15:21:50 +02:00
if job.State == schema.JobStateRunning ||
job.MonitoringStatus == schema.MonitoringStatusRunningOrArchiving ||
!useArchive {
2022-09-13 15:21:50 +02:00
repo, ok := metricDataRepos[job.Cluster]
2022-09-13 15:21:50 +02:00
if !ok {
return fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster), 0, 0
2022-01-20 10:08:50 +01:00
}
2022-09-13 15:21:50 +02:00
if scopes == nil {
scopes = append(scopes, schema.MetricScopeNode)
2022-01-20 10:08:50 +01:00
}
if metrics == nil {
2022-09-13 15:21:50 +02:00
cluster := archive.GetCluster(job.Cluster)
for _, mc := range cluster.MetricConfig {
metrics = append(metrics, mc.Name)
}
}
jd, err = repo.LoadData(job, metrics, scopes, ctx)
if err != nil {
if len(jd) != 0 {
log.Warnf("partial error: %s", err.Error())
// return err, 0, 0 // Reactivating will block archiving on one partial error
2022-09-13 15:21:50 +02:00
} else {
log.Error("Error while loading job data from metric repository")
2022-09-13 15:21:50 +02:00
return err, 0, 0
}
}
2022-09-13 15:21:50 +02:00
size = jd.Size()
} else {
jd, err = archive.GetHandle().LoadJobData(job)
if err != nil {
log.Error("Error while loading job data from archive")
2022-09-13 15:21:50 +02:00
return err, 0, 0
}
2022-09-13 15:21:50 +02:00
// Avoid sending unrequested data to the client:
if metrics != nil || scopes != nil {
if metrics == nil {
metrics = make([]string, 0, len(jd))
for k := range jd {
metrics = append(metrics, k)
}
}
res := schema.JobData{}
for _, metric := range metrics {
if perscope, ok := jd[metric]; ok {
if len(perscope) > 1 {
subset := make(map[schema.MetricScope]*schema.JobMetric)
for _, scope := range scopes {
if jm, ok := perscope[scope]; ok {
subset[scope] = jm
}
2022-04-07 12:28:09 +02:00
}
2022-03-21 13:30:19 +01:00
2022-09-13 15:21:50 +02:00
if len(subset) > 0 {
perscope = subset
}
}
2022-09-13 15:21:50 +02:00
res[metric] = perscope
}
2022-01-20 10:08:50 +01:00
}
2022-09-13 15:21:50 +02:00
jd = res
}
2022-09-13 15:21:50 +02:00
size = jd.Size()
}
ttl = 5 * time.Hour
if job.State == schema.JobStateRunning {
ttl = 2 * time.Minute
}
2022-09-13 15:21:50 +02:00
prepareJobData(job, jd, scopes)
2022-12-08 13:51:44 +01:00
2022-09-13 15:21:50 +02:00
return jd, ttl, size
})
if err, ok := data.(error); ok {
log.Error("Error in returned dataset")
2022-09-13 15:21:50 +02:00
return nil, err
}
2022-09-13 15:21:50 +02:00
return data.(schema.JobData), nil
}
// Used for the jobsFootprint GraphQL-Query. TODO: Rename/Generalize.
func LoadAverages(
job *schema.Job,
metrics []string,
data [][]schema.Float,
2024-02-23 10:08:41 +01:00
ctx context.Context,
) error {
2021-12-17 15:49:22 +01:00
if job.State != schema.JobStateRunning && useArchive {
return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here?
}
repo, ok := metricDataRepos[job.Cluster]
if !ok {
return fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", job.Cluster)
}
2023-07-21 16:33:53 +02:00
stats, err := repo.LoadStats(job, metrics, ctx) // #166 how to handle stats for acc normalizazion?
if err != nil {
log.Errorf("Error while loading statistics for job %v (User %v, Project %v)", job.JobID, job.User, job.Project)
return err
}
for i, m := range metrics {
nodes, ok := stats[m]
if !ok {
data[i] = append(data[i], schema.NaN)
continue
}
sum := 0.0
for _, node := range nodes {
sum += node.Avg
}
data[i] = append(data[i], schema.Float(sum))
}
return nil
}
// Used for the node/system view. Returns a map of nodes to a map of metrics.
func LoadNodeData(
cluster string,
metrics, nodes []string,
scopes []schema.MetricScope,
from, to time.Time,
2024-02-23 10:08:41 +01:00
ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
repo, ok := metricDataRepos[cluster]
if !ok {
return nil, fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster)
}
if metrics == nil {
for _, m := range archive.GetCluster(cluster).MetricConfig {
metrics = append(metrics, m.Name)
}
}
data, err := repo.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
if err != nil {
if len(data) != 0 {
log.Warnf("partial error: %s", err.Error())
} else {
log.Error("Error while loading node data from metric repository")
return nil, err
}
}
if data == nil {
return nil, fmt.Errorf("METRICDATA/METRICDATA > the metric data repository for '%s' does not support this query", cluster)
}
return data, nil
}
func cacheKey(
job *schema.Job,
metrics []string,
2024-02-23 10:08:41 +01:00
scopes []schema.MetricScope,
) string {
// Duration and StartTime do not need to be in the cache key as StartTime is less unique than
// job.ID and the TTL of the cache entry makes sure it does not stay there forever.
2022-01-20 10:08:50 +01:00
return fmt.Sprintf("%d(%s):[%v],[%v]",
job.ID, job.State, metrics, scopes)
}
// For /monitoring/job/<job> and some other places, flops_any and mem_bw need
// to be available at the scope 'node'. If a job has a lot of nodes,
// statisticsSeries should be available so that a min/median/max Graph can be
// used instead of a lot of single lines.
func prepareJobData(
job *schema.Job,
jobData schema.JobData,
2024-02-23 10:08:41 +01:00
scopes []schema.MetricScope,
) {
2022-01-20 10:08:50 +01:00
const maxSeriesSize int = 15
for _, scopes := range jobData {
for _, jm := range scopes {
if jm.StatisticsSeries != nil || len(jm.Series) <= maxSeriesSize {
continue
}
jm.AddStatisticsSeries()
}
}
nodeScopeRequested := false
for _, scope := range scopes {
if scope == schema.MetricScopeNode {
nodeScopeRequested = true
}
}
if nodeScopeRequested {
jobData.AddNodeScope("flops_any")
jobData.AddNodeScope("mem_bw")
}
}
// Writes a running job to the job-archive
func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) {
allMetrics := make([]string, 0)
metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
for _, mc := range metricConfigs {
allMetrics = append(allMetrics, mc.Name)
}
// TODO: Talk about this! What resolutions to store data at...
scopes := []schema.MetricScope{schema.MetricScopeNode}
if job.NumNodes <= 8 {
scopes = append(scopes, schema.MetricScopeCore)
}
jobData, err := LoadData(job, allMetrics, scopes, ctx)
if err != nil {
log.Error("Error wile loading job data for archiving")
return nil, err
}
jobMeta := &schema.JobMeta{
BaseJob: job.BaseJob,
StartTime: job.StartTime.Unix(),
Statistics: make(map[string]schema.JobStatistics),
}
for metric, data := range jobData {
avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
nodeData, ok := data["node"]
if !ok {
// TODO/FIXME: Calc average for non-node metrics as well!
continue
}
for _, series := range nodeData.Series {
avg += series.Statistics.Avg
min = math.Min(min, series.Statistics.Min)
max = math.Max(max, series.Statistics.Max)
}
jobMeta.Statistics[metric] = schema.JobStatistics{
Unit: schema.Unit{
Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
Base: archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
},
Avg: avg / float64(job.NumNodes),
Min: min,
Max: max,
}
}
// If the file based archive is disabled,
// only return the JobMeta structure as the
// statistics in there are needed.
if !useArchive {
return jobMeta, nil
}
2022-09-13 15:21:50 +02:00
return jobMeta, archive.GetHandle().ImportJob(jobMeta, &jobData)
}