ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-09-12 19:53:00 +02:00
Code Issues Packages Releases Activity

Combined metricstore api and functions

Browse Source
This commit is contained in:
Aditya Ujeniya
2025-09-08 11:29:27 +02:00
parent bca176170c
commit 62565b9ae2
26 changed files with 1248 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/avro/avroCheckpoint.go
View File

@@ -19,7 +19,8 @@ import (
"sync/atomic"
"time"
"github.com/ClusterCockpit/cc-lib/util"
"github.com/ClusterCockpit/cc-backend/internal/config"
"github.com/ClusterCockpit/cc-lib/schema"
"github.com/linkedin/goavro/v2"
)
@@ -139,7 +140,7 @@ func getTimestamp(dir string) int64 {
}
}
interval, _ := time.ParseDuration(Keys.Checkpoints.Interval)
interval, _ := time.ParseDuration(config.MetricStoreKeys.Checkpoints.Interval)
updateTime := time.Unix(maxTs, 0).Add(interval).Add(time.Duration(CheckpointBufferMinutes-1) * time.Minute).Unix()
if updateTime < time.Now().Unix() {
@@ -408,7 +409,7 @@ func compareSchema(schemaRead, schemaGen string) (bool, string, error) {
return true, string(mergedSchemaJson), nil
}
func generateSchema(data map[string]util.Float) (string, error) {
func generateSchema(data map[string]schema.Float) (string, error) {
// Define the Avro schema structure
schema := map[string]interface{}{
"type": "record",
@@ -440,7 +441,7 @@ func generateSchema(data map[string]util.Float) (string, error) {
return string(schemaString), nil
}
func generateRecord(data map[string]util.Float) map[string]interface{} {
func generateRecord(data map[string]schema.Float) map[string]interface{} {
record := make(map[string]interface{})
// Iterate through each map in data

5
internal/avro/avroHelper.go
View File

@@ -6,13 +6,14 @@ import (
"strconv"
"sync"
"github.com/ClusterCockpit/cc-backend/internal/config"
)
func DataStaging(wg *sync.WaitGroup, ctx context.Context) {
// AvroPool is a pool of Avro writers.
go func() {
if Keys.Checkpoints.FileFormat == "json" {
if config.MetricStoreKeys.Checkpoints.FileFormat == "json" {
wg.Done() // Mark this goroutine as done
return // Exit the goroutine
}
@@ -28,7 +29,7 @@ func DataStaging(wg *sync.WaitGroup, ctx context.Context) {
return
case val := <-LineProtocolMessages:
//Fetch the frequency of the metric from the global configuration
freq, err := Keys.GetMetricFrequency(val.MetricName)
freq, err := config.MetricStoreKeys.GetMetricFrequency(val.MetricName)
if err != nil {
fmt.Printf("Error fetching metric frequency: %s\n", err)
continue

14
internal/avro/avroStruct.go
View File

@@ -3,7 +3,7 @@ package avro
import (
"sync"
"github.com/ClusterCockpit/cc-lib/util"
"github.com/ClusterCockpit/cc-lib/schema"
)
var (
@@ -20,7 +20,7 @@ type AvroStruct struct {
Cluster string
Node string
Selector []string
Value util.Float
Value schema.Float
Timestamp int64
}
@@ -32,7 +32,7 @@ var avroStore AvroStore
type AvroLevel struct {
children map[string]*AvroLevel
data map[int64]map[string]util.Float
data map[int64]map[string]schema.Float
lock sync.RWMutex
}
@@ -81,7 +81,7 @@ func (l *AvroLevel) findAvroLevelOrCreate(selector []string) *AvroLevel {
}
child = &AvroLevel{
data: make(map[int64]map[string]util.Float, 0),
data: make(map[int64]map[string]schema.Float, 0),
children: nil,
}
@@ -94,7 +94,7 @@ func (l *AvroLevel) findAvroLevelOrCreate(selector []string) *AvroLevel {
return child.findAvroLevelOrCreate(selector[1:])
}
func (l *AvroLevel) addMetric(metricName string, value util.Float, timestamp int64, Freq int) {
func (l *AvroLevel) addMetric(metricName string, value schema.Float, timestamp int64, Freq int) {
l.lock.Lock()
defer l.lock.Unlock()
@@ -104,7 +104,7 @@ func (l *AvroLevel) addMetric(metricName string, value util.Float, timestamp int
if len(l.data) != KeyCounter {
if len(l.data) == 0 {
for i := range KeyCounter {
l.data[timestamp+int64(i*Freq)] = make(map[string]util.Float, 0)
l.data[timestamp+int64(i*Freq)] = make(map[string]schema.Float, 0)
}
} else {
// Get the last timestamp
@@ -115,7 +115,7 @@ func (l *AvroLevel) addMetric(metricName string, value util.Float, timestamp int
}
}
// Create keys for the next KeyCounter timestamps
l.data[lastTs+int64(Freq)] = make(map[string]util.Float, 0)
l.data[lastTs+int64(Freq)] = make(map[string]schema.Float, 0)
}
}