From 50df63a2d2d0bd361e042e63f9867026f88e9abd Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Sun, 21 Dec 2025 13:29:43 +0100 Subject: [PATCH] Remove metricData usage Is replaced by builtin memorystore API --- cmd/cc-backend/main.go | 7 +- configs/config-demo.json | 10 -- configs/config.json | 5 - internal/config/config.go | 9 +- internal/config/schema.go | 17 ++ internal/importer/importer_test.go | 3 - internal/metricdata/metricdata.go | 160 +++++++++--------- internal/metricdata/prometheus.go | 1 + internal/repository/node_test.go | 1 - internal/repository/testdata/job.db | Bin 987136 -> 987136 bytes internal/repository/userConfig_test.go | 1 - internal/taskmanager/metricPullWorker.go | 9 +- internal/taskmanager/taskManager.go | 7 +- .../taskmanager/updateFootprintService.go | 10 +- web/web.go | 2 +- 15 files changed, 116 insertions(+), 126 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index dddbddf..30f7798 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -284,9 +284,9 @@ func initSubsystems() error { } // Initialize metricdata - if err := metricdata.Init(); err != nil { - return fmt.Errorf("initializing metricdata repository: %w", err) - } + // if err := metricdata.Init(); err != nil { + // return fmt.Errorf("initializing metricdata repository: %w", err) + // } // Initialize upstream metricdata repositories for pull worker if err := metricdata.InitUpstreamRepos(); err != nil { @@ -335,7 +335,6 @@ func runServer(ctx context.Context) error { cclog.Debug("Metric store configuration not found, skipping memorystore initialization") } - // Start archiver and task manager archiver.Start(repository.GetJobRepository(), ctx) taskmanager.Start(ccconf.GetPackageConfig("cron"), ccconf.GetPackageConfig("archive")) diff --git a/configs/config-demo.json b/configs/config-demo.json index 58366fb..80a1f65 100644 --- a/configs/config-demo.json +++ b/configs/config-demo.json @@ -37,11 +37,6 @@ "clusters": [ { "name": "fritz", - "metricDataRepository": { - "kind": "cc-metric-store-internal", - "url": "http://localhost:8082", - "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw" - }, "filterRanges": { "numNodes": { "from": 1, @@ -59,11 +54,6 @@ }, { "name": "alex", - "metricDataRepository": { - "kind": "cc-metric-store-internal", - "url": "http://localhost:8082", - "token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJ1c2VyIjoiYWRtaW4iLCJyb2xlcyI6WyJST0xFX0FETUlOIiwiUk9MRV9BTkFMWVNUIiwiUk9MRV9VU0VSIl19.d-3_3FZTsadPjDEdsWrrQ7nS0edMAR4zjl-eK7rJU3HziNBfI9PDHDIpJVHTNN5E5SlLGLFXctWyKAkwhXL-Dw" - }, "filterRanges": { "numNodes": { "from": 1, diff --git a/configs/config.json b/configs/config.json index 88a9e93..f32d48f 100644 --- a/configs/config.json +++ b/configs/config.json @@ -29,11 +29,6 @@ "clusters": [ { "name": "test", - "metricDataRepository": { - "kind": "cc-metric-store", - "url": "http://localhost:8082", - "token": "eyJhbGciOiJF-E-pQBQ" - }, "filterRanges": { "numNodes": { "from": 1, diff --git a/internal/config/config.go b/internal/config/config.go index 1c302f0..b594c6d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -78,6 +78,9 @@ type ProgramConfig struct { // If exists, will enable dynamic zoom in frontend metric plots using the configured values EnableResampling *ResampleConfig `json:"resampling"` + + // Global upstream metric repository configuration for metric pull workers + UpstreamMetricRepository *json.RawMessage `json:"upstreamMetricRepository,omitempty"` } type ResampleConfig struct { @@ -113,10 +116,8 @@ type FilterRanges struct { } type ClusterConfig struct { - Name string `json:"name"` - FilterRanges *FilterRanges `json:"filterRanges"` - MetricDataRepository json.RawMessage `json:"metricDataRepository"` - UpstreamMetricRepository *json.RawMessage `json:"upstreamMetricRepository,omitempty"` + Name string `json:"name"` + FilterRanges *FilterRanges `json:"filterRanges"` } var Clusters []*ClusterConfig diff --git a/internal/config/schema.go b/internal/config/schema.go index 6003a50..f57461d 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -119,6 +119,23 @@ var configSchema = ` } }, "required": ["trigger", "resolutions"] + }, + "upstreamMetricRepository": { + "description": "Global upstream metric repository configuration for metric pull workers", + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": ["influxdb", "prometheus", "cc-metric-store", "cc-metric-store-internal", "test"] + }, + "url": { + "type": "string" + }, + "token": { + "type": "string" + } + }, + "required": ["kind"] } }, "required": ["apiAllowedIPs"] diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go index 470f760..5922e45 100644 --- a/internal/importer/importer_test.go +++ b/internal/importer/importer_test.go @@ -60,7 +60,6 @@ func setup(t *testing.T) *repository.JobRepository { "clusters": [ { "name": "testcluster", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, "filterRanges": { "numNodes": { "from": 1, "to": 64 }, "duration": { "from": 0, "to": 86400 }, @@ -69,7 +68,6 @@ func setup(t *testing.T) *repository.JobRepository { }, { "name": "fritz", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, "filterRanges": { "numNodes": { "from": 1, "to": 944 }, "duration": { "from": 0, "to": 86400 }, @@ -78,7 +76,6 @@ func setup(t *testing.T) *repository.JobRepository { }, { "name": "taurus", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, "filterRanges": { "numNodes": { "from": 1, "to": 4000 }, "duration": { "from": 0, "to": 604800 }, diff --git a/internal/metricdata/metricdata.go b/internal/metricdata/metricdata.go index df33ab4..32ef1cd 100644 --- a/internal/metricdata/metricdata.go +++ b/internal/metricdata/metricdata.go @@ -37,95 +37,91 @@ type MetricDataRepository interface { LoadNodeListData(cluster, subCluster string, nodes, metrics []string, scopes []schema.MetricScope, resolution int, from, to time.Time, ctx context.Context) (map[string]schema.JobData, error) } -var ( - metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{} - upstreamMetricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{} -) +var upstreamMetricDataRepo MetricDataRepository -func Init() error { - for _, cluster := range config.Clusters { - if cluster.MetricDataRepository != nil { - var kind struct { - Kind string `json:"kind"` - } - if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil { - cclog.Warn("Error while unmarshaling raw json MetricDataRepository") - return err - } +// func Init() error { +// for _, cluster := range config.Clusters { +// if cluster.MetricDataRepository != nil { +// var kind struct { +// Kind string `json:"kind"` +// } +// if err := json.Unmarshal(cluster.MetricDataRepository, &kind); err != nil { +// cclog.Warn("Error while unmarshaling raw json MetricDataRepository") +// return err +// } +// +// var mdr MetricDataRepository +// switch kind.Kind { +// case "cc-metric-store": +// mdr = &CCMetricStore{} +// case "prometheus": +// mdr = &PrometheusDataRepository{} +// case "test": +// mdr = &TestMetricDataRepository{} +// default: +// return fmt.Errorf("METRICDATA/METRICDATA > Unknown MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) +// } +// +// if err := mdr.Init(cluster.MetricDataRepository); err != nil { +// cclog.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) +// return err +// } +// metricDataRepos[cluster.Name] = mdr +// } +// } +// return nil +// } - var mdr MetricDataRepository - switch kind.Kind { - case "cc-metric-store": - mdr = &CCMetricStore{} - case "prometheus": - mdr = &PrometheusDataRepository{} - case "test": - mdr = &TestMetricDataRepository{} - default: - return fmt.Errorf("METRICDATA/METRICDATA > Unknown MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) - } +// func GetMetricDataRepo(cluster string) (MetricDataRepository, error) { +// var err error +// repo, ok := metricDataRepos[cluster] +// +// if !ok { +// err = fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster) +// } +// +// return repo, err +// } - if err := mdr.Init(cluster.MetricDataRepository); err != nil { - cclog.Errorf("Error initializing MetricDataRepository %v for cluster %v", kind.Kind, cluster.Name) - return err - } - metricDataRepos[cluster.Name] = mdr - } - } - return nil -} - -func GetMetricDataRepo(cluster string) (MetricDataRepository, error) { - var err error - repo, ok := metricDataRepos[cluster] - - if !ok { - err = fmt.Errorf("METRICDATA/METRICDATA > no metric data repository configured for '%s'", cluster) - } - - return repo, err -} - -// InitUpstreamRepos initializes upstream metric data repositories for the pull worker +// InitUpstreamRepos initializes global upstream metric data repository for the pull worker func InitUpstreamRepos() error { - for _, cluster := range config.Clusters { - if cluster.UpstreamMetricRepository != nil { - var kind struct { - Kind string `json:"kind"` - } - if err := json.Unmarshal(*cluster.UpstreamMetricRepository, &kind); err != nil { - cclog.Warn("Error while unmarshaling raw json UpstreamMetricRepository") - return err - } - - var mdr MetricDataRepository - switch kind.Kind { - case "cc-metric-store": - mdr = &CCMetricStore{} - case "prometheus": - mdr = &PrometheusDataRepository{} - case "test": - mdr = &TestMetricDataRepository{} - default: - return fmt.Errorf("METRICDATA/METRICDATA > Unknown UpstreamMetricRepository %v for cluster %v", kind.Kind, cluster.Name) - } - - if err := mdr.Init(*cluster.UpstreamMetricRepository); err != nil { - cclog.Errorf("Error initializing UpstreamMetricRepository %v for cluster %v", kind.Kind, cluster.Name) - return err - } - upstreamMetricDataRepos[cluster.Name] = mdr - cclog.Infof("Initialized upstream metric repository '%s' for cluster '%s'", kind.Kind, cluster.Name) - } + if config.Keys.UpstreamMetricRepository == nil { + return nil } + + var kind struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(*config.Keys.UpstreamMetricRepository, &kind); err != nil { + cclog.Warn("Error while unmarshaling raw json UpstreamMetricRepository") + return err + } + + var mdr MetricDataRepository + switch kind.Kind { + case "cc-metric-store": + mdr = &CCMetricStore{} + case "prometheus": + mdr = &PrometheusDataRepository{} + case "test": + mdr = &TestMetricDataRepository{} + default: + return fmt.Errorf("METRICDATA/METRICDATA > Unknown UpstreamMetricRepository %v", kind.Kind) + } + + if err := mdr.Init(*config.Keys.UpstreamMetricRepository); err != nil { + cclog.Errorf("Error initializing UpstreamMetricRepository %v", kind.Kind) + return err + } + upstreamMetricDataRepo = mdr + cclog.Infof("Initialized global upstream metric repository '%s'", kind.Kind) return nil } -// GetUpstreamMetricDataRepo returns the upstream metric data repository for a given cluster -func GetUpstreamMetricDataRepo(cluster string) (MetricDataRepository, error) { - repo, ok := upstreamMetricDataRepos[cluster] - if !ok { - return nil, fmt.Errorf("METRICDATA/METRICDATA > no upstream metric data repository configured for '%s'", cluster) +// GetUpstreamMetricDataRepo returns the global upstream metric data repository +func GetUpstreamMetricDataRepo() (MetricDataRepository, error) { + if upstreamMetricDataRepo == nil { + return nil, fmt.Errorf("METRICDATA/METRICDATA > no upstream metric data repository configured") } - return repo, nil + return upstreamMetricDataRepo, nil } diff --git a/internal/metricdata/prometheus.go b/internal/metricdata/prometheus.go index 66c5bc1..288e045 100644 --- a/internal/metricdata/prometheus.go +++ b/internal/metricdata/prometheus.go @@ -2,6 +2,7 @@ // All rights reserved. This file is part of cc-backend. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. + package metricdata import ( diff --git a/internal/repository/node_test.go b/internal/repository/node_test.go index 466f51e..f5a2487 100644 --- a/internal/repository/node_test.go +++ b/internal/repository/node_test.go @@ -42,7 +42,6 @@ func nodeTestSetup(t *testing.T) { "clusters": [ { "name": "testcluster", - "metricDataRepository": {"kind": "test", "url": "bla:8081"}, "filterRanges": { "numNodes": { "from": 1, "to": 64 }, "duration": { "from": 0, "to": 86400 }, diff --git a/internal/repository/testdata/job.db b/internal/repository/testdata/job.db index 5c5a692585a4736a9e0456d1caf7b4878c936f74..575e0f2e5529fe9529aa9e9f10a66e71d4435bd4 100644 GIT binary patch delta 2402 zcma)8VQ3sh6yDvLo!!~y_EnFQjiE^)pamP8-MQS}9W5m7k17ZX{_u~WXN;|KdWUxv zia$2~@^2yfL$Hv41*KFiGzEneAwh{!k&3NSTTE?<5Nj2!Df!`>&F%H}?mQ&CH}CPi z`R2Wu_x3Wkuy1Z*-$zt19o95hJ9D4745bmBX(QuV?MSWB^n-kJw$Yra9&I#dUvJF# z8BNoi%yA7a!8=feJ??dP!F|^CG8Z$)aeFfp9dMYo>!2J~`p4Dv!2`Xqvvk`!YO<_L-s ziwQ#!wKGW)3Cg0_N`ww1DM(Smdiq_fXy3>~glXBF9In*sZ&Vuf^b?B7@)IJ-*cfzJ zF%(jYLeVeYySi4wNRn{OUwU${=eQ>{_yazJFJT^DfobsETkeW`0^98^H`+sY9n}9v zuQ~r6m~JROFtDR(Y!w*wKie;LwuUu`GVFMMHZZXL#zL4$VAMNfeyQ^z=@8B4^ww*U zWLuwIsTVOaG&{_3SN&M?NbR+F>fKEZet}gu4e!F6P=P}*0a?S{#CN#jMxH~tE}e8g z#3Su-h43y!;+%T6LhK$y3eQrA)q{x8V+vtiND$$qc&38PZV21oqY9y2h-&Hw2{m-7 z?H-4(uHCRW=tMJGJbYz_Oey7!Q zF__Ze8k~m(n1knG3ODS9Xp}h`reWA)IsWpYn_oPpU#nEBwV6tD_QjWuR;#0CCw}8X z4^C6~=RG({;U9M4)74QU6%cg=R7dp`Uhl>~IZvnX3T|s~8!NgA!!8-;9Y0YHmafBw z1{(;k!Nd2{@oXoKKVd3_!6^;?#CHB3zbc=?DGYuOp3aaly?q@l<(VmuraY4Jj3F|{ z+bm~=PIvBPu&Tj7a0_n06>N=FJZ=#lw@FS{m(CKyObnVBBryz&E_{8m-5c^)2^9zY1$=e7pI?h)UbWzvIcix0}s6n7hoBW{1~3W$!?XFg z+ZJhC22T9FIvjN^f8m5&hpX@dd