ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-12-31 02:46:16 +01:00
Code Issues Packages Releases Activity

Remove metricData usage

Browse Source 
Is replaced by builtin memorystore API
This commit is contained in:
Jan Eitzinger
2025-12-21 13:29:43 +01:00
parent d23f20f42a
commit 50df63a2d2
15 changed files with 116 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/config/config.go
View File

@@ -78,6 +78,9 @@ type ProgramConfig struct {
// If exists, will enable dynamic zoom in frontend metric plots using the configured values
EnableResampling *ResampleConfig `json:"resampling"`
// Global upstream metric repository configuration for metric pull workers
UpstreamMetricRepository *json.RawMessage `json:"upstreamMetricRepository,omitempty"`
}
type ResampleConfig struct {
@@ -113,10 +116,8 @@ type FilterRanges struct {
}
type ClusterConfig struct {
Name string `json:"name"`
FilterRanges *FilterRanges `json:"filterRanges"`
MetricDataRepository json.RawMessage `json:"metricDataRepository"`
UpstreamMetricRepository *json.RawMessage `json:"upstreamMetricRepository,omitempty"`
Name string `json:"name"`
FilterRanges *FilterRanges `json:"filterRanges"`
}
var Clusters []*ClusterConfig

17
internal/config/schema.go
View File

@@ -119,6 +119,23 @@ var configSchema = `
}
},
"required": ["trigger", "resolutions"]
},
"upstreamMetricRepository": {
"description": "Global upstream metric repository configuration for metric pull workers",
"type": "object",
"properties": {
"kind": {
"type": "string",
"enum": ["influxdb", "prometheus", "cc-metric-store", "cc-metric-store-internal", "test"]
},
"url": {
"type": "string"
},
"token": {
"type": "string"
}
},
"required": ["kind"]
}
},
"required": ["apiAllowedIPs"]

3
internal/importer/importer_test.go
View File

@@ -60,7 +60,6 @@ func setup(t *testing.T) *repository.JobRepository {
"clusters": [
{
"name": "testcluster",
"metricDataRepository": {"kind": "test", "url": "bla:8081"},
"filterRanges": {
"numNodes": { "from": 1, "to": 64 },
"duration": { "from": 0, "to": 86400 },
@@ -69,7 +68,6 @@ func setup(t *testing.T) *repository.JobRepository {
},
{
"name": "fritz",
"metricDataRepository": {"kind": "test", "url": "bla:8081"},
"filterRanges": {
"numNodes": { "from": 1, "to": 944 },
"duration": { "from": 0, "to": 86400 },
@@ -78,7 +76,6 @@ func setup(t *testing.T) *repository.JobRepository {
},
{
"name": "taurus",
"metricDataRepository": {"kind": "test", "url": "bla:8081"},
"filterRanges": {
"numNodes": { "from": 1, "to": 4000 },
"duration": { "from": 0, "to": 604800 },

160
internal/metricdata/metricdata.go
View File

@@ -37,95 +37,91 @@ type MetricDataRepository interface {
LoadNodeListData(cluster, subCluster string, nodes, metrics []string, scopes []schema.MetricScope, resolution int, from, to time.Time, ctx context.Context) (map[string]schema.JobData, error)
}
var (
metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{}
upstreamMetricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{}
)
var upstreamMetricDataRepo MetricDataRepository
func Init() error {
for _, cluster := range config.Clusters {
if cluster.MetricDataRepository != nil {
var kind struct {
Kind string `json:"kind"`
}
if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil {
cclog.Warn("Error while unmarshaling raw json MetricDataRepository")
return err
}
// func Init() error {
// for _, cluster := range config.Clusters {
// if cluster.MetricDataRepository != nil {
// var kind struct {
// Kind string `json:"kind"`
// }
// if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil {
// cclog.Warn("Error while unmarshaling raw json MetricDataRepository")
// return err
// }
//
// var mdr MetricDataRepository
// switch kind.Kind {
// case "cc-metric-store":
// mdr = &CCMetricStore{}
// case "prometheus":
// mdr = &PrometheusDataRepository{}
// case "test":
// mdr = &TestMetricDataRepository{}
// default:
// return fmt.Errorf("METRICDATA/METRICDATA > Unknown MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name)
// }
//
// if err := mdr.Init(cluster.MetricDataRepository); err != nil {
// cclog.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name)
// return err
// }
// metricDataRepos[cluster.Name] = mdr
// }
// }
// return nil
// }
var mdr MetricDataRepository
switch kind.Kind {
case "cc-metric-store":
mdr = &CCMetricStore{}
case "prometheus":
mdr = &PrometheusDataRepository{}
case "test":
mdr = &TestMetricDataRepository{}
default:
return fmt.Errorf("METRICDATA/METRICDATA > Unknown MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name)
}
// func GetMetricDataRepo(cluster string) (MetricDataRepository, error) {
// var err error
// repo, ok := metricDataRepos[cluster]
//
// if !ok {
// err = fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster)
// }
//
// return repo, err
// }
if err := mdr.Init(cluster.MetricDataRepository); err != nil {
cclog.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name)
return err
}
metricDataRepos[cluster.Name] = mdr
}
}
return nil
}
func GetMetricDataRepo(cluster string) (MetricDataRepository, error) {
var err error
repo, ok := metricDataRepos[cluster]
if !ok {
err = fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster)
}
return repo, err
}
// InitUpstreamRepos initializes upstream metric data repositories for the pull worker
// InitUpstreamRepos initializes global upstream metric data repository for the pull worker
func InitUpstreamRepos() error {
for _, cluster := range config.Clusters {
if cluster.UpstreamMetricRepository != nil {
var kind struct {
Kind string `json:"kind"`
}
if err := json.Unmarshal(*cluster.UpstreamMetricRepository, &kind); err != nil {
cclog.Warn("Error while unmarshaling raw json UpstreamMetricRepository")
return err
}
var mdr MetricDataRepository
switch kind.Kind {
case "cc-metric-store":
mdr = &CCMetricStore{}
case "prometheus":
mdr = &PrometheusDataRepository{}
case "test":
mdr = &TestMetricDataRepository{}
default:
return fmt.Errorf("METRICDATA/METRICDATA > Unknown UpstreamMetricRepository %v for cluster %v", kind.Kind, cluster.Name)
}
if err := mdr.Init(*cluster.UpstreamMetricRepository); err != nil {
cclog.Errorf("Error initializing UpstreamMetricRepository %v for cluster %v", kind.Kind, cluster.Name)
return err
}
upstreamMetricDataRepos[cluster.Name] = mdr
cclog.Infof("Initialized upstream metric repository '%s' for cluster '%s'", kind.Kind, cluster.Name)
}
if config.Keys.UpstreamMetricRepository == nil {
return nil
}
var kind struct {
Kind string `json:"kind"`
}
if err := json.Unmarshal(*config.Keys.UpstreamMetricRepository, &kind); err != nil {
cclog.Warn("Error while unmarshaling raw json UpstreamMetricRepository")
return err
}
var mdr MetricDataRepository
switch kind.Kind {
case "cc-metric-store":
mdr = &CCMetricStore{}
case "prometheus":
mdr = &PrometheusDataRepository{}
case "test":
mdr = &TestMetricDataRepository{}
default:
return fmt.Errorf("METRICDATA/METRICDATA > Unknown UpstreamMetricRepository %v", kind.Kind)
}
if err := mdr.Init(*config.Keys.UpstreamMetricRepository); err != nil {
cclog.Errorf("Error initializing UpstreamMetricRepository %v", kind.Kind)
return err
}
upstreamMetricDataRepo = mdr
cclog.Infof("Initialized global upstream metric repository '%s'", kind.Kind)
return nil
}
// GetUpstreamMetricDataRepo returns the upstream metric data repository for a given cluster
func GetUpstreamMetricDataRepo(cluster string) (MetricDataRepository, error) {
repo, ok := upstreamMetricDataRepos[cluster]
if !ok {
return nil, fmt.Errorf("METRICDATA/METRICDATA > no upstream metric data repository configured for '%s'", cluster)
// GetUpstreamMetricDataRepo returns the global upstream metric data repository
func GetUpstreamMetricDataRepo() (MetricDataRepository, error) {
if upstreamMetricDataRepo == nil {
return nil, fmt.Errorf("METRICDATA/METRICDATA > no upstream metric data repository configured")
}
return repo, nil
return upstreamMetricDataRepo, nil
}

1
internal/metricdata/prometheus.go
View File

@@ -2,6 +2,7 @@
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package metricdata
import (

1
internal/repository/node_test.go
View File

@@ -42,7 +42,6 @@ func nodeTestSetup(t *testing.T) {
"clusters": [
{
"name": "testcluster",
"metricDataRepository": {"kind": "test", "url": "bla:8081"},
"filterRanges": {
"numNodes": { "from": 1, "to": 64 },
"duration": { "from": 0, "to": 86400 },

BIN
internal/repository/testdata/job.db vendored
View File

Binary file not shown.

1
internal/repository/userConfig_test.go
View File

@@ -31,7 +31,6 @@ func setupUserTest(t *testing.T) *UserCfgRepo {
"clusters": [
{
"name": "testcluster",
"metricDataRepository": {"kind": "test", "url": "bla:8081"},
"filterRanges": {
"numNodes": { "from": 1, "to": 64 },
"duration": { "from": 0, "to": 86400 },

9
internal/taskmanager/metricPullWorker.go
View File

@@ -11,8 +11,8 @@ import (
"time"
"github.com/ClusterCockpit/cc-backend/internal/config"
"github.com/ClusterCockpit/cc-backend/internal/metricdata"
"github.com/ClusterCockpit/cc-backend/internal/memorystore"
"github.com/ClusterCockpit/cc-backend/internal/metricdata"
"github.com/ClusterCockpit/cc-backend/pkg/archive"
cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
"github.com/ClusterCockpit/cc-lib/schema"
@@ -36,7 +36,6 @@ func RegisterMetricPullWorker() {
// Register one worker per cluster
registered := 0
for _, cluster := range config.Clusters {
cluster := cluster // capture for closure
_, err := s.NewJob(
gocron.DurationJob(d),
@@ -83,10 +82,10 @@ func pullMetricsForCluster(clusterName string) error {
to := time.Now()
from := to.Add(-60 * time.Minute)
// 5. Get upstream backend repository (from separate config)
upstreamRepo, err := metricdata.GetUpstreamMetricDataRepo(clusterName)
// 5. Get upstream backend repository (from global config)
upstreamRepo, err := metricdata.GetUpstreamMetricDataRepo()
if err != nil {
cclog.Debugf("No upstream repository configured for cluster %s, skipping pull", clusterName)
cclog.Debugf("No upstream repository configured, skipping pull for cluster %s", clusterName)
return nil
}

7
internal/taskmanager/taskManager.go
View File

@@ -116,10 +116,11 @@ func Start(cronCfg, archiveConfig json.RawMessage) {
RegisterLdapSyncService(lc.SyncInterval)
}
RegisterMetricPullWorker()
RegisterFootprintWorker()
RegisterUpdateDurationWorker()
RegisterCommitJobService()
RegisterMetricPullWorker()
s.Start()
}
@@ -127,6 +128,8 @@ func Start(cronCfg, archiveConfig json.RawMessage) {
// Shutdown stops the task manager and its scheduler.
func Shutdown() {
if s != nil {
s.Shutdown()
if err := s.Shutdown(); err != nil {
cclog.Errorf("taskmanager Shutdown: error stopping scheduler: %v", err)
}
}
}

10
internal/taskmanager/updateFootprintService.go
View File

@@ -10,7 +10,7 @@ import (
"math"
"time"
"github.com/ClusterCockpit/cc-backend/internal/metricdata"
"github.com/ClusterCockpit/cc-backend/internal/memorystore"
"github.com/ClusterCockpit/cc-backend/pkg/archive"
cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
"github.com/ClusterCockpit/cc-lib/schema"
@@ -58,12 +58,6 @@ func RegisterFootprintWorker() {
allMetrics = append(allMetrics, mc.Name)
}
repo, err := metricdata.GetMetricDataRepo(cluster.Name)
if err != nil {
cclog.Errorf("no metric data repository configured for '%s'", cluster.Name)
continue
}
pendingStatements := []sq.UpdateBuilder{}
for _, job := range jobs {
@@ -72,7 +66,7 @@ func RegisterFootprintWorker() {
sJob := time.Now()
jobStats, err := repo.LoadStats(job, allMetrics, context.Background())
jobStats, err := memorystore.LoadStats(job, allMetrics, context.Background())
if err != nil {
cclog.Errorf("error wile loading job data stats for footprint update: %v", err)
ce++