ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-07-23 12:51:40 +02:00
Code Issues Packages Releases Activity

Port to cc-lib. Extend legal header.

Browse Source
This commit is contained in:
Jan Eitzinger
2025-06-30 12:06:35 +02:00
parent 544fb35121
commit 639e1b9c6d
120 changed files with 1140 additions and 6410 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
internal/repository/stats.go
View File

@@ -1,5 +1,5 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repository
@@ -14,8 +14,8 @@ import (
"github.com/ClusterCockpit/cc-backend/internal/graph/model"
"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
"github.com/ClusterCockpit/cc-backend/pkg/archive"
"github.com/ClusterCockpit/cc-backend/pkg/log"
"github.com/ClusterCockpit/cc-backend/pkg/schema"
cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
"github.com/ClusterCockpit/cc-lib/schema"
sq "github.com/Masterminds/squirrel"
)
@@ -158,7 +158,7 @@ func (r *JobRepository) JobsStatsGrouped(
rows, err := query.RunWith(r.DB).Query()
if err != nil {
log.Warn("Error while querying DB for job statistics")
cclog.Warn("Error while querying DB for job statistics")
return nil, err
}
@@ -169,7 +169,7 @@ func (r *JobRepository) JobsStatsGrouped(
var name sql.NullString
var jobs, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64
if err := rows.Scan(&id, &jobs, &name, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours); err != nil {
log.Warn("Error while scanning rows")
cclog.Warn("Error while scanning rows")
return nil, err
}
@@ -241,7 +241,7 @@ func (r *JobRepository) JobsStatsGrouped(
}
}
log.Debugf("Timer JobsStatsGrouped %s", time.Since(start))
cclog.Debugf("Timer JobsStatsGrouped %s", time.Since(start))
return stats, nil
}
@@ -261,7 +261,7 @@ func (r *JobRepository) JobsStats(
var jobs, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64
if err := row.Scan(&jobs, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours); err != nil {
log.Warn("Error while scanning rows")
cclog.Warn("Error while scanning rows")
return nil, err
}
@@ -287,7 +287,7 @@ func (r *JobRepository) JobsStats(
})
}
log.Debugf("Timer JobStats %s", time.Since(start))
cclog.Debugf("Timer JobStats %s", time.Since(start))
return stats, nil
}
@@ -301,7 +301,7 @@ func LoadJobStat(job *schema.Job, metric string, statType string) float64 {
case "min":
return stats.Min
default:
log.Errorf("Unknown stat type %s", statType)
cclog.Errorf("Unknown stat type %s", statType)
}
}
@@ -322,7 +322,7 @@ func (r *JobRepository) JobCountGrouped(
}
rows, err := query.RunWith(r.DB).Query()
if err != nil {
log.Warn("Error while querying DB for job statistics")
cclog.Warn("Error while querying DB for job statistics")
return nil, err
}
@@ -332,7 +332,7 @@ func (r *JobRepository) JobCountGrouped(
var id sql.NullString
var cnt sql.NullInt64
if err := rows.Scan(&id, &cnt); err != nil {
log.Warn("Error while scanning rows")
cclog.Warn("Error while scanning rows")
return nil, err
}
if id.Valid {
@@ -344,7 +344,7 @@ func (r *JobRepository) JobCountGrouped(
}
}
log.Debugf("Timer JobCountGrouped %s", time.Since(start))
cclog.Debugf("Timer JobCountGrouped %s", time.Since(start))
return stats, nil
}
@@ -364,7 +364,7 @@ func (r *JobRepository) AddJobCountGrouped(
}
rows, err := query.RunWith(r.DB).Query()
if err != nil {
log.Warn("Error while querying DB for job statistics")
cclog.Warn("Error while querying DB for job statistics")
return nil, err
}
@@ -374,7 +374,7 @@ func (r *JobRepository) AddJobCountGrouped(
var id sql.NullString
var cnt sql.NullInt64
if err := rows.Scan(&id, &cnt); err != nil {
log.Warn("Error while scanning rows")
cclog.Warn("Error while scanning rows")
return nil, err
}
if id.Valid {
@@ -393,7 +393,7 @@ func (r *JobRepository) AddJobCountGrouped(
}
}
log.Debugf("Timer AddJobCountGrouped %s", time.Since(start))
cclog.Debugf("Timer AddJobCountGrouped %s", time.Since(start))
return stats, nil
}
@@ -411,7 +411,7 @@ func (r *JobRepository) AddJobCount(
}
rows, err := query.RunWith(r.DB).Query()
if err != nil {
log.Warn("Error while querying DB for job statistics")
cclog.Warn("Error while querying DB for job statistics")
return nil, err
}
@@ -420,7 +420,7 @@ func (r *JobRepository) AddJobCount(
for rows.Next() {
var cnt sql.NullInt64
if err := rows.Scan(&cnt); err != nil {
log.Warn("Error while scanning rows")
cclog.Warn("Error while scanning rows")
return nil, err
}
@@ -438,7 +438,7 @@ func (r *JobRepository) AddJobCount(
}
}
log.Debugf("Timer AddJobCount %s", time.Since(start))
cclog.Debugf("Timer AddJobCount %s", time.Since(start))
return stats, nil
}
@@ -479,29 +479,29 @@ func (r *JobRepository) AddHistograms(
value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as %s) as value`, time.Now().Unix(), targetBinSize, castType)
stat.HistDuration, err = r.jobsDurationStatisticsHistogram(ctx, value, filter, targetBinSize, &targetBinCount)
if err != nil {
log.Warn("Error while loading job statistics histogram: job duration")
cclog.Warn("Error while loading job statistics histogram: job duration")
return nil, err
}
stat.HistNumNodes, err = r.jobsStatisticsHistogram(ctx, "job.num_nodes as value", filter)
if err != nil {
log.Warn("Error while loading job statistics histogram: num nodes")
cclog.Warn("Error while loading job statistics histogram: num nodes")
return nil, err
}
stat.HistNumCores, err = r.jobsStatisticsHistogram(ctx, "job.num_hwthreads as value", filter)
if err != nil {
log.Warn("Error while loading job statistics histogram: num hwthreads")
cclog.Warn("Error while loading job statistics histogram: num hwthreads")
return nil, err
}
stat.HistNumAccs, err = r.jobsStatisticsHistogram(ctx, "job.num_acc as value", filter)
if err != nil {
log.Warn("Error while loading job statistics histogram: num acc")
cclog.Warn("Error while loading job statistics histogram: num acc")
return nil, err
}
log.Debugf("Timer AddHistograms %s", time.Since(start))
cclog.Debugf("Timer AddHistograms %s", time.Since(start))
return stat, nil
}
@@ -520,7 +520,7 @@ func (r *JobRepository) AddMetricHistograms(
if f.State != nil {
if len(f.State) == 1 && f.State[0] == "running" {
stat.HistMetrics = r.runningJobsMetricStatisticsHistogram(ctx, metrics, filter, targetBinCount)
log.Debugf("Timer AddMetricHistograms %s", time.Since(start))
cclog.Debugf("Timer AddMetricHistograms %s", time.Since(start))
return stat, nil
}
}
@@ -530,13 +530,13 @@ func (r *JobRepository) AddMetricHistograms(
for _, m := range metrics {
metricHisto, err := r.jobsMetricStatisticsHistogram(ctx, m, filter, targetBinCount)
if err != nil {
log.Warnf("Error while loading job metric statistics histogram: %s", m)
cclog.Warnf("Error while loading job metric statistics histogram: %s", m)
continue
}
stat.HistMetrics = append(stat.HistMetrics, metricHisto)
}
log.Debugf("Timer AddMetricHistograms %s", time.Since(start))
cclog.Debugf("Timer AddMetricHistograms %s", time.Since(start))
return stat, nil
}
@@ -560,7 +560,7 @@ func (r *JobRepository) jobsStatisticsHistogram(
rows, err := query.GroupBy("value").RunWith(r.DB).Query()
if err != nil {
log.Error("Error while running query")
cclog.Error("Error while running query")
return nil, err
}
@@ -569,13 +569,13 @@ func (r *JobRepository) jobsStatisticsHistogram(
for rows.Next() {
point := model.HistoPoint{}
if err := rows.Scan(&point.Value, &point.Count); err != nil {
log.Warn("Error while scanning rows")
cclog.Warn("Error while scanning rows")
return nil, err
}
points = append(points, &point)
}
log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
return points, nil
}
@@ -607,7 +607,7 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
rows, err := query.GroupBy("value").RunWith(r.DB).Query()
if err != nil {
log.Error("Error while running query")
cclog.Error("Error while running query")
return nil, err
}
@@ -615,7 +615,7 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
for rows.Next() {
point := model.HistoPoint{}
if err := rows.Scan(&point.Value, &point.Count); err != nil {
log.Warn("Error while scanning rows")
cclog.Warn("Error while scanning rows")
return nil, err
}
@@ -630,7 +630,7 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
}
}
log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
return points, nil
}
@@ -652,7 +652,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
peak = metricConfig.Peak
unit = metricConfig.Unit.Prefix + metricConfig.Unit.Base
footprintStat = metricConfig.Footprint
log.Debugf("Cluster %s filter found with peak %f for %s", *f.Cluster.Eq, peak, metric)
cclog.Debugf("Cluster %s filter found with peak %f for %s", *f.Cluster.Eq, peak, metric)
}
}
@@ -674,7 +674,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
}
}
// log.Debugf("Metric %s, Peak %f, Unit %s", metric, peak, unit)
// cclog.Debugf("Metric %s, Peak %f, Unit %s", metric, peak, unit)
// Make bins, see https://jereze.com/code/sql-histogram/ (Modified here)
start := time.Now()
@@ -709,7 +709,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
rows, err := mainQuery.RunWith(r.DB).Query()
if err != nil {
log.Errorf("Error while running mainQuery: %s", err)
cclog.Errorf("Error while running mainQuery: %s", err)
return nil, err
}
@@ -726,7 +726,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
for rows.Next() { // Fill Count if Bin-No. Matches (Not every Bin exists in DB!)
rpoint := model.MetricHistoPoint{}
if err := rows.Scan(&rpoint.Bin, &rpoint.Count); err != nil { // Required for Debug: &rpoint.Min, &rpoint.Max
log.Warnf("Error while scanning rows for %s", metric)
cclog.Warnf("Error while scanning rows for %s", metric)
return nil, err // FIXME: Totally bricks cc-backend if returned and if all metrics requested?
}
@@ -736,10 +736,10 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
e.Count = rpoint.Count
// Only Required For Debug: Check DB returned Min/Max against Backend Init above
// if rpoint.Min != nil {
// log.Warnf(">>>> Bin %d Min Set For %s to %d (Init'd with: %d)", *e.Bin, metric, *rpoint.Min, *e.Min)
// cclog.Warnf(">>>> Bin %d Min Set For %s to %d (Init'd with: %d)", *e.Bin, metric, *rpoint.Min, *e.Min)
// }
// if rpoint.Max != nil {
// log.Warnf(">>>> Bin %d Max Set For %s to %d (Init'd with: %d)", *e.Bin, metric, *rpoint.Max, *e.Max)
// cclog.Warnf(">>>> Bin %d Max Set For %s to %d (Init'd with: %d)", *e.Bin, metric, *rpoint.Max, *e.Max)
// }
break
}
@@ -749,7 +749,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
result := model.MetricHistoPoints{Metric: metric, Unit: unit, Stat: &footprintStat, Data: points}
log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
return &result, nil
}
@@ -762,11 +762,11 @@ func (r *JobRepository) runningJobsMetricStatisticsHistogram(
// Get Jobs
jobs, err := r.QueryJobs(ctx, filters, &model.PageRequest{Page: 1, ItemsPerPage: 500 + 1}, nil)
if err != nil {
log.Errorf("Error while querying jobs for footprint: %s", err)
cclog.Errorf("Error while querying jobs for footprint: %s", err)
return nil
}
if len(jobs) > 500 {
log.Errorf("too many jobs matched (max: %d)", 500)
cclog.Errorf("too many jobs matched (max: %d)", 500)
return nil
}
@@ -782,7 +782,7 @@ func (r *JobRepository) runningJobsMetricStatisticsHistogram(
}
if err := metricDataDispatcher.LoadAverages(job, metrics, avgs, ctx); err != nil {
log.Errorf("Error while loading averages for histogram: %s", err)
cclog.Errorf("Error while loading averages for histogram: %s", err)
return nil
}
}