mirror of
https://github.com/ClusterCockpit/cc-backend
synced 2025-12-13 02:46:16 +01:00
Jan Eitzinger
8d44ac90ad
Remove unblocking default in select Add shutdown handler with context and timeout
106 lines
3.4 KiB
Go
106 lines
3.4 KiB
Go
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
|
|
// All rights reserved. This file is part of cc-backend.
|
|
// Use of this source code is governed by a MIT-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package archiver
|
|
|
|
import (
|
|
"context"
|
|
"math"
|
|
|
|
"github.com/ClusterCockpit/cc-backend/internal/config"
|
|
"github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher"
|
|
"github.com/ClusterCockpit/cc-backend/pkg/archive"
|
|
cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
|
|
"github.com/ClusterCockpit/cc-lib/schema"
|
|
)
|
|
|
|
// ArchiveJob archives a completed job's metric data to the configured archive backend.
|
|
//
|
|
// This function performs the following operations:
|
|
// 1. Loads all metric data for the job from the metric data repository
|
|
// 2. Calculates job-level statistics (avg, min, max) for each metric
|
|
// 3. Stores the job metadata and metric data to the archive backend
|
|
//
|
|
// Metric data is retrieved at the highest available resolution (typically 60s)
|
|
// for the following scopes:
|
|
// - Node scope (always)
|
|
// - Core scope (for jobs with ≤8 nodes, to reduce data volume)
|
|
// - Accelerator scope (if job used accelerators)
|
|
//
|
|
// The function respects context cancellation. If ctx is cancelled (e.g., during
|
|
// shutdown timeout), the operation will be interrupted and return an error.
|
|
//
|
|
// Parameters:
|
|
// - job: The job to archive (must be a completed job)
|
|
// - ctx: Context for cancellation and timeout control
|
|
//
|
|
// Returns:
|
|
// - *schema.Job with populated Statistics field
|
|
// - error if data loading or archiving fails
|
|
//
|
|
// If config.Keys.DisableArchive is true, only job statistics are calculated
|
|
// and returned (no data is written to archive backend).
|
|
func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
|
|
allMetrics := make([]string, 0)
|
|
metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
|
|
for _, mc := range metricConfigs {
|
|
allMetrics = append(allMetrics, mc.Name)
|
|
}
|
|
|
|
scopes := []schema.MetricScope{schema.MetricScopeNode}
|
|
// FIXME: Add a config option for this
|
|
if job.NumNodes <= 8 {
|
|
// This will add the native scope if core scope is not available
|
|
scopes = append(scopes, schema.MetricScopeCore)
|
|
}
|
|
|
|
if job.NumAcc > 0 {
|
|
scopes = append(scopes, schema.MetricScopeAccelerator)
|
|
}
|
|
|
|
jobData, err := metricDataDispatcher.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s)
|
|
if err != nil {
|
|
cclog.Error("Error wile loading job data for archiving")
|
|
return nil, err
|
|
}
|
|
|
|
job.Statistics = make(map[string]schema.JobStatistics)
|
|
|
|
for metric, data := range jobData {
|
|
avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
|
|
nodeData, ok := data["node"]
|
|
if !ok {
|
|
// This should never happen ?
|
|
continue
|
|
}
|
|
|
|
for _, series := range nodeData.Series {
|
|
avg += series.Statistics.Avg
|
|
min = math.Min(min, series.Statistics.Min)
|
|
max = math.Max(max, series.Statistics.Max)
|
|
}
|
|
|
|
// Round AVG Result to 2 Digits
|
|
job.Statistics[metric] = schema.JobStatistics{
|
|
Unit: schema.Unit{
|
|
Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
|
|
Base: archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
|
|
},
|
|
Avg: (math.Round((avg/float64(job.NumNodes))*100) / 100),
|
|
Min: min,
|
|
Max: max,
|
|
}
|
|
}
|
|
|
|
// If the file based archive is disabled,
|
|
// only return the JobMeta structure as the
|
|
// statistics in there are needed.
|
|
if config.Keys.DisableArchive {
|
|
return job, nil
|
|
}
|
|
|
|
return job, archive.GetHandle().ImportJob(job, &jobData)
|
|
}
|