Refactor package structure

Builds but not tested

internal/metricdata/archive.go

@@ -1,261 +0,0 @@
-// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
-// All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-package metricdata
-
-import (
-	"bufio"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"math"
-	"os"
-	"path"
-	"path/filepath"
-	"strconv"
-	"time"
-
-	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-backend/pkg/schema"
-)
-
-// For a given job, return the path of the `data.json`/`meta.json` file.
-// TODO: Implement Issue ClusterCockpit/ClusterCockpit#97
-func getPath(job *schema.Job, file string, checkLegacy bool) (string, error) {
-	lvl1, lvl2 := fmt.Sprintf("%d", job.JobID/1000), fmt.Sprintf("%03d", job.JobID%1000)
-	if !checkLegacy {
-		return filepath.Join(JobArchivePath, job.Cluster, lvl1, lvl2, strconv.FormatInt(job.StartTime.Unix(), 10), file), nil
-	}
-
-	legacyPath := filepath.Join(JobArchivePath, job.Cluster, lvl1, lvl2, file)
-	if _, err := os.Stat(legacyPath); errors.Is(err, os.ErrNotExist) {
-		return filepath.Join(JobArchivePath, job.Cluster, lvl1, lvl2, strconv.FormatInt(job.StartTime.Unix(), 10), file), nil
-	}
-
-	return legacyPath, nil
-}
-
-// Assuming job is completed/archived, return the jobs metric data.
-func loadFromArchive(job *schema.Job) (schema.JobData, error) {
-	filename, err := getPath(job, "data.json", true)
-	if err != nil {
-		return nil, err
-	}
-
-	data := cache.Get(filename, func() (value interface{}, ttl time.Duration, size int) {
-		f, err := os.Open(filename)
-		if err != nil {
-			return err, 0, 1000
-		}
-		defer f.Close()
-
-		var data schema.JobData
-		if err := json.NewDecoder(bufio.NewReader(f)).Decode(&data); err != nil {
-			return err, 0, 1000
-		}
-
-		return data, 1 * time.Hour, data.Size()
-	})
-
-	if err, ok := data.(error); ok {
-		return nil, err
-	}
-
-	return data.(schema.JobData), nil
-}
-
-func loadMetaJson(job *schema.Job) (*schema.JobMeta, error) {
-	filename, err := getPath(job, "meta.json", true)
-	if err != nil {
-		return nil, err
-	}
-
-	bytes, err := os.ReadFile(filename)
-	if err != nil {
-		return nil, err
-	}
-
-	var metaFile schema.JobMeta = schema.JobMeta{
-		BaseJob: schema.JobDefaults,
-	}
-	if err := json.Unmarshal(bytes, &metaFile); err != nil {
-		return nil, err
-	}
-
-	return &metaFile, nil
-}
-
-// If the job is archived, find its `meta.json` file and override the tags list
-// in that JSON file. If the job is not archived, nothing is done.
-func UpdateTags(job *schema.Job, tags []*schema.Tag) error {
-	if job.State == schema.JobStateRunning {
-		return nil
-	}
-
-	filename, err := getPath(job, "meta.json", true)
-	if err != nil {
-		return err
-	}
-
-	f, err := os.Open(filename)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-
-	var metaFile schema.JobMeta = schema.JobMeta{
-		BaseJob: schema.JobDefaults,
-	}
-	if err := json.NewDecoder(f).Decode(&metaFile); err != nil {
-		return err
-	}
-	f.Close()
-
-	metaFile.Tags = make([]*schema.Tag, 0)
-	for _, tag := range tags {
-		metaFile.Tags = append(metaFile.Tags, &schema.Tag{
-			Name: tag.Name,
-			Type: tag.Type,
-		})
-	}
-
-	bytes, err := json.Marshal(metaFile)
-	if err != nil {
-		return err
-	}
-
-	return os.WriteFile(filename, bytes, 0644)
-}
-
-// Helper to metricdata.LoadAverages().
-func loadAveragesFromArchive(job *schema.Job, metrics []string, data [][]schema.Float) error {
-	metaFile, err := loadMetaJson(job)
-	if err != nil {
-		return err
-	}
-
-	for i, m := range metrics {
-		if stat, ok := metaFile.Statistics[m]; ok {
-			data[i] = append(data[i], schema.Float(stat.Avg))
-		} else {
-			data[i] = append(data[i], schema.NaN)
-		}
-	}
-
-	return nil
-}
-
-func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) {
-	metaFile, err := loadMetaJson(job)
-	if err != nil {
-		return nil, err
-	}
-
-	return metaFile.Statistics, nil
-}
-
-// Writes a running job to the job-archive
-func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) {
-	allMetrics := make([]string, 0)
-	metricConfigs := config.GetCluster(job.Cluster).MetricConfig
-	for _, mc := range metricConfigs {
-		allMetrics = append(allMetrics, mc.Name)
-	}
-
-	// TODO: Talk about this! What resolutions to store data at...
-	scopes := []schema.MetricScope{schema.MetricScopeNode}
-	if job.NumNodes <= 8 {
-		scopes = append(scopes, schema.MetricScopeCore)
-	}
-
-	jobData, err := LoadData(job, allMetrics, scopes, ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	jobMeta := &schema.JobMeta{
-		BaseJob:    job.BaseJob,
-		StartTime:  job.StartTime.Unix(),
-		Statistics: make(map[string]schema.JobStatistics),
-	}
-
-	for metric, data := range jobData {
-		avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
-		nodeData, ok := data["node"]
-		if !ok {
-			// TODO/FIXME: Calc average for non-node metrics as well!
-			continue
-		}
-
-		for _, series := range nodeData.Series {
-			avg += series.Statistics.Avg
-			min = math.Min(min, series.Statistics.Min)
-			max = math.Max(max, series.Statistics.Max)
-		}
-
-		jobMeta.Statistics[metric] = schema.JobStatistics{
-			Unit: config.GetMetricConfig(job.Cluster, metric).Unit,
-			Avg:  avg / float64(job.NumNodes),
-			Min:  min,
-			Max:  max,
-		}
-	}
-
-	// If the file based archive is disabled,
-	// only return the JobMeta structure as the
-	// statistics in there are needed.
-	if !useArchive {
-		return jobMeta, nil
-	}
-
-	dir, err := getPath(job, "", false)
-	if err != nil {
-		return nil, err
-	}
-
-	return jobMeta, writeFiles(dir, jobMeta, &jobData)
-}
-
-func writeFiles(dir string, jobMeta *schema.JobMeta, jobData *schema.JobData) error {
-	if err := os.MkdirAll(dir, 0777); err != nil {
-		return err
-	}
-
-	f, err := os.Create(path.Join(dir, "meta.json"))
-	if err != nil {
-		return err
-	}
-	if err := json.NewEncoder(f).Encode(jobMeta); err != nil {
-		return err
-	}
-	if err := f.Close(); err != nil {
-		return err
-	}
-
-	f, err = os.Create(path.Join(dir, "data.json"))
-	if err != nil {
-		return err
-	}
-	if err := json.NewEncoder(f).Encode(jobData); err != nil {
-		return err
-	}
-	return f.Close()
-}
-
-// Used to import a non-running job into the job-archive.
-func ImportJob(job *schema.JobMeta, jobData *schema.JobData) error {
-	dir, err := getPath(&schema.Job{
-		BaseJob:       job.BaseJob,
-		StartTimeUnix: job.StartTime,
-		StartTime:     time.Unix(job.StartTime, 0),
-	}, "", false)
-	if err != nil {
-		return err
-	}
-
-	return writeFiles(dir, job, jobData)
-}

internal/metricdata/cc-metric-store.go

@@ -15,7 +15,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
 	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
@@ -149,7 +149,7 @@ func (ccms *CCMetricStore) doRequest(ctx context.Context, body *ApiQueryRequest)
 }
 
 func (ccms *CCMetricStore) LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error) {
-	topology := config.GetSubCluster(job.Cluster, job.SubCluster).Topology
+	topology := archive.GetSubCluster(job.Cluster, job.SubCluster).Topology
 	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes)
 	if err != nil {
 		return nil, err
@@ -175,7 +175,7 @@ func (ccms *CCMetricStore) LoadData(job *schema.Job, metrics []string, scopes []
 		query := req.Queries[i]
 		metric := ccms.toLocalName(query.Metric)
 		scope := assignedScope[i]
-		mc := config.GetMetricConfig(job.Cluster, metric)
+		mc := archive.GetMetricConfig(job.Cluster, metric)
 		if _, ok := jobData[metric]; !ok {
 			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
 		}
@@ -252,12 +252,12 @@ var (
 
 func (ccms *CCMetricStore) buildQueries(job *schema.Job, metrics []string, scopes []schema.MetricScope) ([]ApiQuery, []schema.MetricScope, error) {
 	queries := make([]ApiQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
-	topology := config.GetSubCluster(job.Cluster, job.SubCluster).Topology
+	topology := archive.GetSubCluster(job.Cluster, job.SubCluster).Topology
 	assignedScope := []schema.MetricScope{}
 
 	for _, metric := range metrics {
 		remoteName := ccms.toRemoteName(metric)
-		mc := config.GetMetricConfig(job.Cluster, metric)
+		mc := archive.GetMetricConfig(job.Cluster, metric)
 		if mc == nil {
 			// return nil, fmt.Errorf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
 			// log.Printf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
@@ -584,7 +584,7 @@ func (ccms *CCMetricStore) LoadNodeData(cluster string, metrics, nodes []string,
 			data[query.Hostname] = hostdata
 		}
 
-		mc := config.GetMetricConfig(cluster, metric)
+		mc := archive.GetMetricConfig(cluster, metric)
 		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
 			Unit:     mc.Unit,
 			Scope:    schema.MetricScopeNode,

internal/metricdata/influxdb-v2.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
 	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
 	influxdb2Api "github.com/influxdata/influxdb-client-go/v2/api"
@@ -124,7 +124,7 @@ func (idb *InfluxDBv2DataRepository) LoadData(job *schema.Job, metrics []string,
 		for _, metric := range metrics {
 			jobMetric, ok := jobData[metric]
 			if !ok {
-				mc := config.GetMetricConfig(job.Cluster, metric)
+				mc := archive.GetMetricConfig(job.Cluster, metric)
 				jobMetric = map[schema.MetricScope]*schema.JobMetric{
 					scope: { // uses scope var from above!
 						Unit:             mc.Unit,

internal/metricdata/metricdata.go

@@ -8,9 +8,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"math"
 	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/pkg/archive"
 	"github.com/ClusterCockpit/cc-backend/pkg/log"
 	"github.com/ClusterCockpit/cc-backend/pkg/lrucache"
 	"github.com/ClusterCockpit/cc-backend/pkg/schema"
@@ -33,14 +35,11 @@ type MetricDataRepository interface {
 
 var metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{}
 
-var JobArchivePath string
-
 var useArchive bool
 
-func Init(jobArchivePath string, disableArchive bool) error {
+func Init(disableArchive bool) error {
 	useArchive = !disableArchive
-	JobArchivePath = jobArchivePath
-	for _, cluster := range config.Clusters {
+	for _, cluster := range config.Keys.Clusters {
 		if cluster.MetricDataRepository != nil {
 			var kind struct {
 				Kind string `json:"kind"`
@@ -73,97 +72,88 @@ func Init(jobArchivePath string, disableArchive bool) error {
 var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024)
 
 // Fetches the metric data for a job.
-func LoadData(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context) (schema.JobData, error) {
-	data := cache.Get(cacheKey(job, metrics, scopes), func() (_ interface{}, ttl time.Duration, size int) {
-		var jd schema.JobData
-		var err error
-		if job.State == schema.JobStateRunning ||
-			job.MonitoringStatus == schema.MonitoringStatusRunningOrArchiving ||
-			!useArchive {
-			repo, ok := metricDataRepos[job.Cluster]
-			if !ok {
-				return fmt.Errorf("no metric data repository configured for '%s'", job.Cluster), 0, 0
-			}
+func LoadData(job *schema.Job,
+	metrics []string,
+	scopes []schema.MetricScope,
+	ctx context.Context) (schema.JobData, error) {
+	var jd schema.JobData
+	var err error
 
-			if scopes == nil {
-				scopes = append(scopes, schema.MetricScopeNode)
-			}
+	if job.State == schema.JobStateRunning ||
+		job.MonitoringStatus == schema.MonitoringStatusRunningOrArchiving ||
+		!useArchive {
+		repo, ok := metricDataRepos[job.Cluster]
 
+		if !ok {
+			return nil, fmt.Errorf("no metric data repository configured for '%s'", job.Cluster)
+		}
+
+		if scopes == nil {
+			scopes = append(scopes, schema.MetricScopeNode)
+		}
+
+		if metrics == nil {
+			cluster := archive.GetCluster(job.Cluster)
+			for _, mc := range cluster.MetricConfig {
+				metrics = append(metrics, mc.Name)
+			}
+		}
+
+		jd, err = repo.LoadData(job, metrics, scopes, ctx)
+		if err != nil {
+			if len(jd) != 0 {
+				log.Errorf("partial error: %s", err.Error())
+			} else {
+				return nil, err
+			}
+		}
+	} else {
+		jd, err = archive.GetHandle().LoadJobData(job)
+		if err != nil {
+			return nil, err
+		}
+
+		// Avoid sending unrequested data to the client:
+		if metrics != nil || scopes != nil {
 			if metrics == nil {
-				cluster := config.GetCluster(job.Cluster)
-				for _, mc := range cluster.MetricConfig {
-					metrics = append(metrics, mc.Name)
+				metrics = make([]string, 0, len(jd))
+				for k := range jd {
+					metrics = append(metrics, k)
 				}
 			}
 
-			jd, err = repo.LoadData(job, metrics, scopes, ctx)
-			if err != nil {
-				if len(jd) != 0 {
-					log.Errorf("partial error: %s", err.Error())
-				} else {
-					return err, 0, 0
-				}
-			}
-			size = jd.Size()
-		} else {
-			jd, err = loadFromArchive(job)
-			if err != nil {
-				return err, 0, 0
-			}
-
-			// Avoid sending unrequested data to the client:
-			if metrics != nil || scopes != nil {
-				if metrics == nil {
-					metrics = make([]string, 0, len(jd))
-					for k := range jd {
-						metrics = append(metrics, k)
-					}
-				}
-
-				res := schema.JobData{}
-				for _, metric := range metrics {
-					if perscope, ok := jd[metric]; ok {
-						if len(perscope) > 1 {
-							subset := make(map[schema.MetricScope]*schema.JobMetric)
-							for _, scope := range scopes {
-								if jm, ok := perscope[scope]; ok {
-									subset[scope] = jm
-								}
-							}
-
-							if len(subset) > 0 {
-								perscope = subset
+			res := schema.JobData{}
+			for _, metric := range metrics {
+				if perscope, ok := jd[metric]; ok {
+					if len(perscope) > 1 {
+						subset := make(map[schema.MetricScope]*schema.JobMetric)
+						for _, scope := range scopes {
+							if jm, ok := perscope[scope]; ok {
+								subset[scope] = jm
 							}
 						}
 
-						res[metric] = perscope
+						if len(subset) > 0 {
+							perscope = subset
+						}
 					}
+
+					res[metric] = perscope
 				}
-				jd = res
 			}
-			size = 1 // loadFromArchive() caches in the same cache.
+			jd = res
 		}
-
-		ttl = 5 * time.Hour
-		if job.State == schema.JobStateRunning {
-			ttl = 2 * time.Minute
-		}
-
-		prepareJobData(job, jd, scopes)
-		return jd, ttl, size
-	})
-
-	if err, ok := data.(error); ok {
-		return nil, err
 	}
 
-	return data.(schema.JobData), nil
+	prepareJobData(job, jd, scopes)
+
+	return jd, nil
 }
 
 // Used for the jobsFootprint GraphQL-Query. TODO: Rename/Generalize.
 func LoadAverages(job *schema.Job, metrics []string, data [][]schema.Float, ctx context.Context) error {
 	if job.State != schema.JobStateRunning && useArchive {
-		return loadAveragesFromArchive(job, metrics, data)
+		return archive.LoadAveragesFromArchive(job, metrics, data)
 	}
 
 	repo, ok := metricDataRepos[job.Cluster]
@@ -201,7 +191,7 @@ func LoadNodeData(cluster string, metrics, nodes []string, scopes []schema.Metri
 	}
 
 	if metrics == nil {
-		for _, m := range config.GetCluster(cluster).MetricConfig {
+		for _, m := range archive.GetCluster(cluster).MetricConfig {
 			metrics = append(metrics, m.Name)
 		}
 	}
@@ -256,3 +246,60 @@ func prepareJobData(job *schema.Job, jobData schema.JobData, scopes []schema.Met
 		jobData.AddNodeScope("mem_bw")
 	}
 }
+
+// Writes a running job to the job-archive
+func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) {
+	allMetrics := make([]string, 0)
+	metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
+	for _, mc := range metricConfigs {
+		allMetrics = append(allMetrics, mc.Name)
+	}
+
+	// TODO: Talk about this! What resolutions to store data at...
+	scopes := []schema.MetricScope{schema.MetricScopeNode}
+	if job.NumNodes <= 8 {
+		scopes = append(scopes, schema.MetricScopeCore)
+	}
+
+	jobData, err := LoadData(job, allMetrics, scopes, ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	jobMeta := &schema.JobMeta{
+		BaseJob:    job.BaseJob,
+		StartTime:  job.StartTime.Unix(),
+		Statistics: make(map[string]schema.JobStatistics),
+	}
+
+	for metric, data := range jobData {
+		avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
+		nodeData, ok := data["node"]
+		if !ok {
+			// TODO/FIXME: Calc average for non-node metrics as well!
+			continue
+		}
+
+		for _, series := range nodeData.Series {
+			avg += series.Statistics.Avg
+			min = math.Min(min, series.Statistics.Min)
+			max = math.Max(max, series.Statistics.Max)
+		}
+
+		jobMeta.Statistics[metric] = schema.JobStatistics{
+			Unit: archive.GetMetricConfig(job.Cluster, metric).Unit,
+			Avg:  avg / float64(job.NumNodes),
+			Min:  min,
+			Max:  max,
+		}
+	}
+
+	// If the file based archive is disabled,
+	// only return the JobMeta structure as the
+	// statistics in there are needed.
+	if !useArchive {
+		return jobMeta, nil
+	}
+
+	return jobMeta, archive.Import(jobMeta, &jobData)
+}