ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-11-01 00:15:05 +01:00
Code Issues Packages Releases Activity

Move avro into memorystore. Refactor

Browse Source 
Does not compile
This commit is contained in:
Jan Eitzinger
2025-10-18 08:30:42 +02:00
parent 6efd6334bb
commit bac51891b7
19 changed files with 456 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
internal/memorystore/api.go
View File

@@ -2,6 +2,7 @@
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
@@ -47,7 +48,7 @@ type ErrorResponse struct {
Error string `json:"error"` // Error Message
}
type ApiMetricData struct {
type APIMetricData struct {
Error *string `json:"error,omitempty"`
Data schema.FloatArray `json:"data,omitempty"`
From int64 `json:"from"`
@@ -69,7 +70,7 @@ func handleError(err error, statusCode int, rw http.ResponseWriter) {
}
// TODO: Optimize this, just like the stats endpoint!
func (data *ApiMetricData) AddStats() {
func (data *APIMetricData) AddStats() {
n := 0
sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
for _, x := range data.Data {
@@ -93,7 +94,7 @@ func (data *ApiMetricData) AddStats() {
}
}
func (data *ApiMetricData) ScaleBy(f schema.Float) {
func (data *APIMetricData) ScaleBy(f schema.Float) {
if f == 0 || f == 1 {
return
}
@@ -106,7 +107,7 @@ func (data *ApiMetricData) ScaleBy(f schema.Float) {
}
}
func (data *ApiMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metric string) {
func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metric string) {
minfo, ok := ms.Metrics[metric]
if !ok {
return
@@ -115,7 +116,7 @@ func (data *ApiMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metr
if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
ndata := make([]schema.Float, 0, padfront+len(data.Data))
for i := 0; i < padfront; i++ {
for range padfront {
ndata = append(ndata, schema.NaN)
}
for j := 0; j < len(data.Data); j++ {
@@ -218,9 +219,9 @@ func HandleWrite(rw http.ResponseWriter, r *http.Request) {
rw.WriteHeader(http.StatusOK)
}
type ApiQueryRequest struct {
type APIQueryRequest struct {
Cluster string `json:"cluster"`
Queries []ApiQuery `json:"queries"`
Queries []APIQuery `json:"queries"`
ForAllNodes []string `json:"for-all-nodes"`
From int64 `json:"from"`
To int64 `json:"to"`
@@ -229,12 +230,12 @@ type ApiQueryRequest struct {
WithPadding bool `json:"with-padding"`
}
type ApiQueryResponse struct {
Queries []ApiQuery `json:"queries,omitempty"`
Results [][]ApiMetricData `json:"results"`
type APIQueryResponse struct {
Queries []APIQuery `json:"queries,omitempty"`
Results [][]APIMetricData `json:"results"`
}
type ApiQuery struct {
type APIQuery struct {
Type *string `json:"type,omitempty"`
SubType *string `json:"subtype,omitempty"`
Metric string `json:"metric"`
@@ -246,22 +247,21 @@ type ApiQuery struct {
Aggregate bool `json:"aggreg"`
}
func FetchData(req ApiQueryRequest) (*ApiQueryResponse, error) {
func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
req.WithData = true
req.WithData = true
req.WithData = true
ms := GetMemoryStore()
response := ApiQueryResponse{
Results: make([][]ApiMetricData, 0, len(req.Queries)),
response := APIQueryResponse{
Results: make([][]APIMetricData, 0, len(req.Queries)),
}
if req.ForAllNodes != nil {
nodes := ms.ListChildren([]string{req.Cluster})
for _, node := range nodes {
for _, metric := range req.ForAllNodes {
q := ApiQuery{
q := APIQuery{
Metric: metric,
Hostname: node,
}
@@ -300,21 +300,21 @@ func FetchData(req ApiQueryRequest) (*ApiQueryResponse, error) {
}
sels = append(sels, sel)
} else {
for _, typeId := range query.TypeIds {
for _, typeID := range query.TypeIds {
if query.SubType != nil {
for _, subTypeId := range query.SubTypeIds {
for _, subTypeID := range query.SubTypeIds {
sels = append(sels, util.Selector{
{String: req.Cluster},
{String: query.Hostname},
{String: *query.Type + typeId},
{String: *query.SubType + subTypeId},
{String: *query.Type + typeID},
{String: *query.SubType + subTypeID},
})
}
} else {
sels = append(sels, util.Selector{
{String: req.Cluster},
{String: query.Hostname},
{String: *query.Type + typeId},
{String: *query.Type + typeID},
})
}
}
@@ -323,12 +323,11 @@ func FetchData(req ApiQueryRequest) (*ApiQueryResponse, error) {
// log.Printf("query: %#v\n", query)
// log.Printf("sels: %#v\n", sels)
var err error
res := make([]ApiMetricData, 0, len(sels))
res := make([]APIMetricData, 0, len(sels))
for _, sel := range sels {
data := ApiMetricData{}
data := APIMetricData{}
data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
if err != nil {
msg := err.Error()
data.Error = &msg

12
internal/memorystore/archive.go
View File

@@ -2,6 +2,7 @@
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
@@ -18,14 +19,13 @@ import (
"sync/atomic"
"time"
"github.com/ClusterCockpit/cc-backend/internal/config"
cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
)
func Archiving(wg *sync.WaitGroup, ctx context.Context) {
go func() {
defer wg.Done()
d, err := time.ParseDuration(config.MetricStoreKeys.Archive.Interval)
d, err := time.ParseDuration(Keys.Archive.Interval)
if err != nil {
log.Fatalf("[METRICSTORE]> error parsing archive interval duration: %v\n", err)
}
@@ -46,8 +46,8 @@ func Archiving(wg *sync.WaitGroup, ctx context.Context) {
case <-ticks:
t := time.Now().Add(-d)
log.Printf("[METRICSTORE]> start archiving checkpoints (older than %s)...\n", t.Format(time.RFC3339))
n, err := ArchiveCheckpoints(config.MetricStoreKeys.Checkpoints.RootDir,
config.MetricStoreKeys.Archive.RootDir, t.Unix(), config.MetricStoreKeys.Archive.DeleteInstead)
n, err := ArchiveCheckpoints(Keys.Checkpoints.RootDir,
Keys.Archive.RootDir, t.Unix(), Keys.Archive.DeleteInstead)
if err != nil {
log.Printf("[METRICSTORE]> archiving failed: %s\n", err.Error())
@@ -59,7 +59,7 @@ func Archiving(wg *sync.WaitGroup, ctx context.Context) {
}()
}
var ErrNoNewData error = errors.New("all data already archived")
var ErrNoNewArchiveData error = errors.New("all data already archived")
// ZIP all checkpoint files older than `from` together and write them to the `archiveDir`,
// deleting them from the `checkpointsDir`.
@@ -129,7 +129,7 @@ func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead
return 0, err
}
extension := config.MetricStoreKeys.Checkpoints.FileFormat
extension := Keys.Checkpoints.FileFormat
files, err := findFiles(entries, from, extension, false)
if err != nil {
return 0, err

475
internal/memorystore/avroCheckpoint.go Normal file
View File

@@ -0,0 +1,475 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"path"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/ClusterCockpit/cc-lib/schema"
"github.com/linkedin/goavro/v2"
)
var NumAvroWorkers int = 4
var ErrNoNewData error = errors.New("no data in the pool")
func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) {
levels := make([]*AvroLevel, 0)
selectors := make([][]string, 0)
as.root.lock.RLock()
// Cluster
for sel1, l1 := range as.root.children {
l1.lock.RLock()
// Node
for sel2, l2 := range l1.children {
l2.lock.RLock()
// Frequency
for sel3, l3 := range l2.children {
levels = append(levels, l3)
selectors = append(selectors, []string{sel1, sel2, sel3})
}
l2.lock.RUnlock()
}
l1.lock.RUnlock()
}
as.root.lock.RUnlock()
type workItem struct {
level *AvroLevel
dir string
selector []string
}
n, errs := int32(0), int32(0)
var wg sync.WaitGroup
wg.Add(NumAvroWorkers)
work := make(chan workItem, NumAvroWorkers*2)
for range NumAvroWorkers {
go func() {
defer wg.Done()
for workItem := range work {
from := getTimestamp(workItem.dir)
if err := workItem.level.toCheckpoint(workItem.dir, from, dumpAll); err != nil {
if err == ErrNoNewArchiveData {
continue
}
log.Printf("error while checkpointing %#v: %s", workItem.selector, err.Error())
atomic.AddInt32(&errs, 1)
} else {
atomic.AddInt32(&n, 1)
}
}
}()
}
for i := range len(levels) {
dir := path.Join(dir, path.Join(selectors[i]...))
work <- workItem{
level: levels[i],
dir: dir,
selector: selectors[i],
}
}
close(work)
wg.Wait()
if errs > 0 {
return int(n), fmt.Errorf("%d errors happend while creating avro checkpoints (%d successes)", errs, n)
}
return int(n), nil
}
// getTimestamp returns the timestamp from the directory name
func getTimestamp(dir string) int64 {
// Extract the resolution and timestamp from the directory name
// The existing avro file will be in epoch timestamp format
// iterate over all the files in the directory and find the maximum timestamp
// and return it
resolution := path.Base(dir)
dir = path.Dir(dir)
files, err := os.ReadDir(dir)
if err != nil {
return 0
}
var maxTS int64 = 0
if len(files) == 0 {
return 0
}
for _, file := range files {
if file.IsDir() {
continue
}
name := file.Name()
if len(name) < 5 || !strings.HasSuffix(name, ".avro") || !strings.HasPrefix(name, resolution+"_") {
continue
}
ts, err := strconv.ParseInt(name[strings.Index(name, "_")+1:len(name)-5], 10, 64)
if err != nil {
fmt.Printf("error while parsing timestamp: %s\n", err.Error())
continue
}
if ts > maxTS {
maxTS = ts
}
}
interval, _ := time.ParseDuration(Keys.Checkpoints.Interval)
updateTime := time.Unix(maxTS, 0).Add(interval).Add(time.Duration(CheckpointBufferMinutes-1) * time.Minute).Unix()
if updateTime < time.Now().Unix() {
return 0
}
return maxTS
}
func (l *AvroLevel) toCheckpoint(dir string, from int64, dumpAll bool) error {
l.lock.Lock()
defer l.lock.Unlock()
// fmt.Printf("Checkpointing directory: %s\n", dir)
// filepath contains the resolution
intRes, _ := strconv.Atoi(path.Base(dir))
// find smallest overall timestamp in l.data map and delete it from l.data
minTS := int64(1<<63 - 1)
for ts, dat := range l.data {
if ts < minTS && len(dat) != 0 {
minTS = ts
}
}
if from == 0 && minTS != int64(1<<63-1) {
from = minTS
}
if from == 0 {
return ErrNoNewArchiveData
}
var schema string
var codec *goavro.Codec
recordList := make([]map[string]any, 0)
var f *os.File
filePath := dir + fmt.Sprintf("_%d.avro", from)
var err error
fp_, err_ := os.Stat(filePath)
if errors.Is(err_, os.ErrNotExist) {
err = os.MkdirAll(path.Dir(dir), 0o755)
if err != nil {
return fmt.Errorf("failed to create directory: %v", err)
}
} else if fp_.Size() != 0 {
f, err = os.Open(filePath)
if err != nil {
return fmt.Errorf("failed to open existing avro file: %v", err)
}
br := bufio.NewReader(f)
reader, err := goavro.NewOCFReader(br)
if err != nil {
return fmt.Errorf("failed to create OCF reader: %v", err)
}
codec = reader.Codec()
schema = codec.Schema()
f.Close()
}
timeRef := time.Now().Add(time.Duration(-CheckpointBufferMinutes+1) * time.Minute).Unix()
if dumpAll {
timeRef = time.Now().Unix()
}
// Empty values
if len(l.data) == 0 {
// we checkpoint avro files every 60 seconds
repeat := 60 / intRes
for range repeat {
recordList = append(recordList, make(map[string]any))
}
}
readFlag := true
for ts := range l.data {
flag := false
if ts < timeRef {
data := l.data[ts]
schemaGen, err := generateSchema(data)
if err != nil {
return err
}
flag, schema, err = compareSchema(schema, schemaGen)
if err != nil {
return fmt.Errorf("failed to compare read and generated schema: %v", err)
}
if flag && readFlag && !errors.Is(err_, os.ErrNotExist) {
f.Close()
f, err = os.Open(filePath)
if err != nil {
return fmt.Errorf("failed to open Avro file: %v", err)
}
br := bufio.NewReader(f)
ocfReader, err := goavro.NewOCFReader(br)
if err != nil {
return fmt.Errorf("failed to create OCF reader while changing schema: %v", err)
}
for ocfReader.Scan() {
record, err := ocfReader.Read()
if err != nil {
return fmt.Errorf("failed to read record: %v", err)
}
recordList = append(recordList, record.(map[string]any))
}
f.Close()
err = os.Remove(filePath)
if err != nil {
return fmt.Errorf("failed to delete file: %v", err)
}
readFlag = false
}
codec, err = goavro.NewCodec(schema)
if err != nil {
return fmt.Errorf("failed to create codec after merged schema: %v", err)
}
recordList = append(recordList, generateRecord(data))
delete(l.data, ts)
}
}
if len(recordList) == 0 {
return ErrNoNewArchiveData
}
f, err = os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0o644)
if err != nil {
return fmt.Errorf("failed to append new avro file: %v", err)
}
// fmt.Printf("Codec : %#v\n", codec)
writer, err := goavro.NewOCFWriter(goavro.OCFConfig{
W: f,
Codec: codec,
CompressionName: goavro.CompressionDeflateLabel,
})
if err != nil {
return fmt.Errorf("failed to create OCF writer: %v", err)
}
// Append the new record
if err := writer.Append(recordList); err != nil {
return fmt.Errorf("failed to append record: %v", err)
}
f.Close()
return nil
}
func compareSchema(schemaRead, schemaGen string) (bool, string, error) {
var genSchema, readSchema AvroSchema
if schemaRead == "" {
return false, schemaGen, nil
}
// Unmarshal the schema strings into AvroSchema structs
if err := json.Unmarshal([]byte(schemaGen), &genSchema); err != nil {
return false, "", fmt.Errorf("failed to parse generated schema: %v", err)
}
if err := json.Unmarshal([]byte(schemaRead), &readSchema); err != nil {
return false, "", fmt.Errorf("failed to parse read schema: %v", err)
}
sort.Slice(genSchema.Fields, func(i, j int) bool {
return genSchema.Fields[i].Name < genSchema.Fields[j].Name
})
sort.Slice(readSchema.Fields, func(i, j int) bool {
return readSchema.Fields[i].Name < readSchema.Fields[j].Name
})
// Check if schemas are identical
schemasEqual := true
if len(genSchema.Fields) <= len(readSchema.Fields) {
for i := range genSchema.Fields {
if genSchema.Fields[i].Name != readSchema.Fields[i].Name {
schemasEqual = false
break
}
}
// If schemas are identical, return the read schema
if schemasEqual {
return false, schemaRead, nil
}
}
// Create a map to hold unique fields from both schemas
fieldMap := make(map[string]AvroField)
// Add fields from the read schema
for _, field := range readSchema.Fields {
fieldMap[field.Name] = field
}
// Add or update fields from the generated schema
for _, field := range genSchema.Fields {
fieldMap[field.Name] = field
}
// Create a union schema by collecting fields from the map
var mergedFields []AvroField
for _, field := range fieldMap {
mergedFields = append(mergedFields, field)
}
// Sort fields by name for consistency
sort.Slice(mergedFields, func(i, j int) bool {
return mergedFields[i].Name < mergedFields[j].Name
})
// Create the merged schema
mergedSchema := AvroSchema{
Type: "record",
Name: genSchema.Name,
Fields: mergedFields,
}
// Check if schemas are identical
schemasEqual = len(mergedSchema.Fields) == len(readSchema.Fields)
if schemasEqual {
for i := range mergedSchema.Fields {
if mergedSchema.Fields[i].Name != readSchema.Fields[i].Name {
schemasEqual = false
break
}
}
if schemasEqual {
return false, schemaRead, nil
}
}
// Marshal the merged schema back to JSON
mergedSchemaJSON, err := json.Marshal(mergedSchema)
if err != nil {
return false, "", fmt.Errorf("failed to marshal merged schema: %v", err)
}
return true, string(mergedSchemaJSON), nil
}
func generateSchema(data map[string]schema.Float) (string, error) {
// Define the Avro schema structure
schema := map[string]any{
"type": "record",
"name": "DataRecord",
"fields": []map[string]any{},
}
fieldTracker := make(map[string]struct{})
for key := range data {
if _, exists := fieldTracker[key]; !exists {
key = correctKey(key)
field := map[string]any{
"name": key,
"type": "double",
"default": -1.0,
}
schema["fields"] = append(schema["fields"].([]map[string]any), field)
fieldTracker[key] = struct{}{}
}
}
schemaString, err := json.Marshal(schema)
if err != nil {
return "", fmt.Errorf("failed to marshal schema: %v", err)
}
return string(schemaString), nil
}
func generateRecord(data map[string]schema.Float) map[string]any {
record := make(map[string]any)
// Iterate through each map in data
for key, value := range data {
key = correctKey(key)
// Set the value in the record
// avro only accepts basic types
record[key] = value.Double()
}
return record
}
func correctKey(key string) string {
// Replace any invalid characters in the key
// For example, replace spaces with underscores
key = strings.ReplaceAll(key, ":", "___")
key = strings.ReplaceAll(key, ".", "__")
return key
}
func ReplaceKey(key string) string {
// Replace any invalid characters in the key
// For example, replace spaces with underscores
key = strings.ReplaceAll(key, "___", ":")
key = strings.ReplaceAll(key, "__", ".")
return key
}

83
internal/memorystore/avroHelper.go Normal file
View File

@@ -0,0 +1,83 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
"context"
"log"
"slices"
"strconv"
"sync"
)
func DataStaging(wg *sync.WaitGroup, ctx context.Context) {
// AvroPool is a pool of Avro writers.
go func() {
if Keys.Checkpoints.FileFormat == "json" {
wg.Done() // Mark this goroutine as done
return // Exit the goroutine
}
defer wg.Done()
var avroLevel *AvroLevel
oldSelector := make([]string, 0)
for {
select {
case <-ctx.Done():
return
case val := <-LineProtocolMessages:
// Fetch the frequency of the metric from the global configuration
freq, err := GetMetricFrequency(val.MetricName)
if err != nil {
log.Printf("Error fetching metric frequency: %s\n", err)
continue
}
metricName := ""
for _, selectorName := range val.Selector {
metricName += selectorName + Delimiter
}
metricName += val.MetricName
// Create a new selector for the Avro level
// The selector is a slice of strings that represents the path to the
// Avro level. It is created by appending the cluster, node, and metric
// name to the selector.
var selector []string
selector = append(selector, val.Cluster, val.Node, strconv.FormatInt(freq, 10))
if !testEq(oldSelector, selector) {
// Get the Avro level for the metric
avroLevel = avroStore.root.findAvroLevelOrCreate(selector)
// If the Avro level is nil, create a new one
if avroLevel == nil {
log.Printf("Error creating or finding the level with cluster : %s, node : %s, metric : %s\n", val.Cluster, val.Node, val.MetricName)
}
oldSelector = slices.Clone(selector)
}
avroLevel.addMetric(metricName, val.Value, val.Timestamp, int(freq))
}
}
}()
}
func testEq(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}

168
internal/memorystore/avroStruct.go Normal file
View File

@@ -0,0 +1,168 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
"sync"
"github.com/ClusterCockpit/cc-lib/schema"
)
var (
LineProtocolMessages = make(chan *AvroStruct)
Delimiter = "ZZZZZ"
)
// CheckpointBufferMinutes should always be in minutes.
// Its controls the amount of data to hold for given amount of time.
var CheckpointBufferMinutes = 3
type AvroStruct struct {
MetricName string
Cluster string
Node string
Selector []string
Value schema.Float
Timestamp int64
}
type AvroStore struct {
root AvroLevel
}
var avroStore AvroStore
type AvroLevel struct {
children map[string]*AvroLevel
data map[int64]map[string]schema.Float
lock sync.RWMutex
}
type AvroField struct {
Name string `json:"name"`
Type any `json:"type"`
Default any `json:"default,omitempty"`
}
type AvroSchema struct {
Type string `json:"type"`
Name string `json:"name"`
Fields []AvroField `json:"fields"`
}
func (l *AvroLevel) findAvroLevelOrCreate(selector []string) *AvroLevel {
if len(selector) == 0 {
return l
}
// Allow concurrent reads:
l.lock.RLock()
var child *AvroLevel
var ok bool
if l.children == nil {
// Children map needs to be created...
l.lock.RUnlock()
} else {
child, ok := l.children[selector[0]]
l.lock.RUnlock()
if ok {
return child.findAvroLevelOrCreate(selector[1:])
}
}
// The level does not exist, take write lock for unqiue access:
l.lock.Lock()
// While this thread waited for the write lock, another thread
// could have created the child node.
if l.children != nil {
child, ok = l.children[selector[0]]
if ok {
l.lock.Unlock()
return child.findAvroLevelOrCreate(selector[1:])
}
}
child = &AvroLevel{
data: make(map[int64]map[string]schema.Float, 0),
children: nil,
}
if l.children != nil {
l.children[selector[0]] = child
} else {
l.children = map[string]*AvroLevel{selector[0]: child}
}
l.lock.Unlock()
return child.findAvroLevelOrCreate(selector[1:])
}
func (l *AvroLevel) addMetric(metricName string, value schema.Float, timestamp int64, Freq int) {
l.lock.Lock()
defer l.lock.Unlock()
KeyCounter := int(CheckpointBufferMinutes * 60 / Freq)
// Create keys in advance for the given amount of time
if len(l.data) != KeyCounter {
if len(l.data) == 0 {
for i := range KeyCounter {
l.data[timestamp+int64(i*Freq)] = make(map[string]schema.Float, 0)
}
} else {
// Get the last timestamp
var lastTS int64
for ts := range l.data {
if ts > lastTS {
lastTS = ts
}
}
// Create keys for the next KeyCounter timestamps
l.data[lastTS+int64(Freq)] = make(map[string]schema.Float, 0)
}
}
closestTS := int64(0)
minDiff := int64(Freq) + 1 // Start with diff just outside the valid range
found := false
// Iterate over timestamps and choose the one which is within range.
// Since its epoch time, we check if the difference is less than 60 seconds.
for ts, dat := range l.data {
// Check if timestamp is within range
diff := timestamp - ts
if diff < -int64(Freq) || diff > int64(Freq) {
continue
}
// Metric already present at this timestamp — skip
if _, ok := dat[metricName]; ok {
continue
}
// Check if this is the closest timestamp so far
if Abs(diff) < minDiff {
minDiff = Abs(diff)
closestTS = ts
found = true
}
}
if found {
l.data[closestTS][metricName] = value
}
}
func GetAvroStore() *AvroStore {
return &avroStore
}
// Abs returns the absolute value of x.
func Abs(x int64) int64 {
if x < 0 {
return -x
}
return x
}

13
internal/memorystore/buffer.go
View File

@@ -1,3 +1,8 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
@@ -12,14 +17,14 @@ import (
// in the buffer chain will be created if needed so that no copying
// of data or reallocation needs to happen on writes.
const (
BUFFER_CAP int = 512
BufferCap int = 512
)
// So that we can reuse allocations
var bufferPool sync.Pool = sync.Pool{
New: func() interface{} {
New: func() any {
return &buffer{
data: make([]schema.Float, 0, BUFFER_CAP),
data: make([]schema.Float, 0, BufferCap),
}
},
}
@@ -192,7 +197,7 @@ func (b *buffer) free(t int64) (delme bool, n int) {
n += m
if delme {
b.prev.next = nil
if cap(b.prev.data) == BUFFER_CAP {
if cap(b.prev.data) == BufferCap {
bufferPool.Put(b.prev)
}
b.prev = nil

65
internal/memorystore/checkpoint.go
View File

@@ -1,3 +1,8 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
@@ -19,8 +24,6 @@ import (
"sync/atomic"
"time"
"github.com/ClusterCockpit/cc-backend/internal/avro"
"github.com/ClusterCockpit/cc-backend/internal/config"
"github.com/ClusterCockpit/cc-lib/schema"
"github.com/linkedin/goavro/v2"
)
@@ -44,12 +47,12 @@ var lastCheckpoint time.Time
func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
lastCheckpoint = time.Now()
if config.MetricStoreKeys.Checkpoints.FileFormat == "json" {
if Keys.Checkpoints.FileFormat == "json" {
ms := GetMemoryStore()
go func() {
defer wg.Done()
d, err := time.ParseDuration(config.MetricStoreKeys.Checkpoints.Interval)
d, err := time.ParseDuration(Keys.Checkpoints.Interval)
if err != nil {
log.Fatal(err)
}
@@ -70,7 +73,7 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
case <-ticks:
log.Printf("[METRICSTORE]> start checkpointing (starting at %s)...\n", lastCheckpoint.Format(time.RFC3339))
now := time.Now()
n, err := ms.ToCheckpoint(config.MetricStoreKeys.Checkpoints.RootDir,
n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir,
lastCheckpoint.Unix(), now.Unix())
if err != nil {
log.Printf("[METRICSTORE]> checkpointing failed: %s\n", err.Error())
@@ -89,9 +92,9 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
select {
case <-ctx.Done():
return
case <-time.After(time.Duration(avro.CheckpointBufferMinutes) * time.Minute):
case <-time.After(time.Duration(CheckpointBufferMinutes) * time.Minute):
// This is the first tick untill we collect the data for given minutes.
avro.GetAvroStore().ToCheckpoint(config.MetricStoreKeys.Checkpoints.RootDir, false)
GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false)
// log.Printf("Checkpointing %d avro files", count)
}
@@ -109,7 +112,7 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
return
case <-ticks:
// Regular ticks of 1 minute to write data.
avro.GetAvroStore().ToCheckpoint(config.MetricStoreKeys.Checkpoints.RootDir, false)
GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false)
// log.Printf("Checkpointing %d avro files", count)
}
}
@@ -176,7 +179,7 @@ func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
for workItem := range work {
if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
if err == ErrNoNewData {
if err == ErrNoNewArchiveData {
continue
}
@@ -219,7 +222,7 @@ func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFil
}
for metric, minfo := range m.Metrics {
b := l.metrics[minfo.Offset]
b := l.metrics[minfo.offset]
if b == nil {
continue
}
@@ -278,7 +281,7 @@ func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
}
if cf == nil {
return ErrNoNewData
return ErrNoNewArchiveData
}
filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
@@ -376,7 +379,7 @@ done:
func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
if _, err := os.Stat(dir); os.IsNotExist(err) {
// The directory does not exist, so create it using os.MkdirAll()
err := os.MkdirAll(dir, 0755) // 0755 sets the permissions for the directory
err := os.MkdirAll(dir, 0o755) // 0755 sets the permissions for the directory
if err != nil {
log.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
}
@@ -384,7 +387,7 @@ func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
}
// Config read (replace with your actual config read)
fileFormat := config.MetricStoreKeys.Checkpoints.FileFormat
fileFormat := Keys.Checkpoints.FileFormat
if fileFormat == "" {
fileFormat = "avro"
}
@@ -445,10 +448,10 @@ func (l *Level) loadAvroFile(m *MemoryStore, f *os.File, from int64) error {
return fmt.Errorf("[METRICSTORE]> error while reading avro file (resolution parsing) : %s", err)
}
from_timestamp, err := strconv.ParseInt(fileName[strings.Index(fileName, "_")+1:len(fileName)-5], 10, 64)
fromTimestamp, err := strconv.ParseInt(fileName[strings.Index(fileName, "_")+1:len(fileName)-5], 10, 64)
// Same logic according to lineprotocol
from_timestamp -= (resolution / 2)
fromTimestamp -= (resolution / 2)
if err != nil {
return fmt.Errorf("[METRICSTORE]> error converting timestamp from the avro file : %s", err)
@@ -472,7 +475,7 @@ func (l *Level) loadAvroFile(m *MemoryStore, f *os.File, from int64) error {
return fmt.Errorf("[METRICSTORE]> error while reading avro file : %s", err)
}
record, ok := datum.(map[string]interface{})
record, ok := datum.(map[string]any)
if !ok {
panic("[METRICSTORE]> failed to assert datum as map[string]interface{}")
}
@@ -484,16 +487,16 @@ func (l *Level) loadAvroFile(m *MemoryStore, f *os.File, from int64) error {
recordCounter += 1
}
to := (from_timestamp + (recordCounter / (60 / resolution) * 60))
to := (fromTimestamp + (recordCounter / (60 / resolution) * 60))
if to < from {
return nil
}
for key, floatArray := range metricsData {
metricName := avro.ReplaceKey(key)
metricName := ReplaceKey(key)
if strings.Contains(metricName, avro.Delimiter) {
subString := strings.Split(metricName, avro.Delimiter)
if strings.Contains(metricName, Delimiter) {
subString := strings.Split(metricName, Delimiter)
lvl := l
@@ -517,12 +520,12 @@ func (l *Level) loadAvroFile(m *MemoryStore, f *os.File, from int64) error {
}
leafMetricName := subString[len(subString)-1]
err = lvl.createBuffer(m, leafMetricName, floatArray, from_timestamp, resolution)
err = lvl.createBuffer(m, leafMetricName, floatArray, fromTimestamp, resolution)
if err != nil {
return fmt.Errorf("[METRICSTORE]> error while creating buffers from avroReader : %s", err)
}
} else {
err = l.createBuffer(m, metricName, floatArray, from_timestamp, resolution)
err = l.createBuffer(m, metricName, floatArray, fromTimestamp, resolution)
if err != nil {
return fmt.Errorf("[METRICSTORE]> error while creating buffers from avroReader : %s", err)
}
@@ -551,9 +554,9 @@ func (l *Level) createBuffer(m *MemoryStore, metricName string, floatArray schem
// return errors.New("Unkown metric: " + name)
}
prev := l.metrics[minfo.Offset]
prev := l.metrics[minfo.offset]
if prev == nil {
l.metrics[minfo.Offset] = b
l.metrics[minfo.offset] = b
} else {
if prev.start > b.start {
return errors.New("wooops")
@@ -573,12 +576,12 @@ func (l *Level) createBuffer(m *MemoryStore, metricName string, floatArray schem
prev.data = prev.data[0:len(prev.data):len(prev.data)]
}
}
l.metrics[minfo.Offset] = b
l.metrics[minfo.offset] = b
return nil
}
func (l *Level) loadJsonFile(m *MemoryStore, f *os.File, from int64) error {
func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
br := bufio.NewReader(f)
cf := &CheckpointFile{}
if err := json.NewDecoder(br).Decode(cf); err != nil {
@@ -615,9 +618,9 @@ func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
// return errors.New("Unkown metric: " + name)
}
prev := l.metrics[minfo.Offset]
prev := l.metrics[minfo.offset]
if prev == nil {
l.metrics[minfo.Offset] = b
l.metrics[minfo.offset] = b
} else {
if prev.start > b.start {
return errors.New("wooops")
@@ -626,7 +629,7 @@ func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
b.prev = prev
prev.next = b
}
l.metrics[minfo.Offset] = b
l.metrics[minfo.offset] = b
}
if len(cf.Children) > 0 && l.children == nil {
@@ -690,7 +693,7 @@ func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64, extension
}
loaders := map[string]func(*MemoryStore, *os.File, int64) error{
"json": l.loadJsonFile,
"json": l.loadJSONFile,
"avro": l.loadAvroFile,
}
@@ -736,7 +739,7 @@ func findFiles(direntries []fs.DirEntry, t int64, extension string, findMoreRece
})
filenames := make([]string, 0)
for i := 0; i < len(direntries); i++ {
for i := range direntries {
e := direntries[i]
ts1 := nums[e.Name()]

134
internal/memorystore/config.go Normal file
View File

@@ -0,0 +1,134 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
"bytes"
"encoding/json"
"fmt"
"github.com/ClusterCockpit/cc-backend/internal/config"
cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
)
var InternalCCMSFlag bool = false
type MetricStoreConfig struct {
Checkpoints struct {
FileFormat string `json:"file-format"`
Interval string `json:"interval"`
RootDir string `json:"directory"`
Restore string `json:"restore"`
} `json:"checkpoints"`
Debug struct {
DumpToFile string `json:"dump-to-file"`
EnableGops bool `json:"gops"`
} `json:"debug"`
RetentionInMemory string `json:"retention-in-memory"`
Archive struct {
Interval string `json:"interval"`
RootDir string `json:"directory"`
DeleteInstead bool `json:"delete-instead"`
} `json:"archive"`
Nats []*NatsConfig `json:"nats"`
}
type NatsConfig struct {
// Address of the nats server
Address string `json:"address"`
// Username/Password, optional
Username string `json:"username"`
Password string `json:"password"`
// Creds file path
Credsfilepath string `json:"creds-file-path"`
Subscriptions []struct {
// Channel name
SubscribeTo string `json:"subscribe-to"`
// Allow lines without a cluster tag, use this as default, optional
ClusterTag string `json:"cluster-tag"`
} `json:"subscriptions"`
}
var Keys MetricStoreConfig
// AggregationStrategy for aggregation over multiple values at different cpus/sockets/..., not time!
type AggregationStrategy int
const (
NoAggregation AggregationStrategy = iota
SumAggregation
AvgAggregation
)
func AssignAggregationStratergy(str string) (AggregationStrategy, error) {
switch str {
case "":
return NoAggregation, nil
case "sum":
return SumAggregation, nil
case "avg":
return AvgAggregation, nil
default:
return NoAggregation, fmt.Errorf("[METRICSTORE]> unknown aggregation strategy: %s", str)
}
}
type MetricConfig struct {
// Interval in seconds at which measurements are stored
Frequency int64
// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
Aggregation AggregationStrategy
// Private, used internally...
offset int
}
var Metrics map[string]MetricConfig
func InitMetricStore(rawConfig json.RawMessage) {
if rawConfig != nil {
config.Validate(configSchema, rawConfig)
dec := json.NewDecoder(bytes.NewReader(rawConfig))
// dec.DisallowUnknownFields()
if err := dec.Decode(&Keys); err != nil {
cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
}
}
}
func GetMetricFrequency(metricName string) (int64, error) {
if metric, ok := Metrics[metricName]; ok {
return metric.Frequency, nil
}
return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
}
// AddMetric adds logic to add metrics. Redundant metrics should be updated with max frequency.
// use metric.Name to check if the metric already exists.
// if not, add it to the Metrics map.
func AddMetric(name string, metric MetricConfig) error {
if Metrics == nil {
Metrics = make(map[string]MetricConfig, 0)
}
if existingMetric, ok := Metrics[name]; ok {
if existingMetric.Frequency != metric.Frequency {
if existingMetric.Frequency < metric.Frequency {
existingMetric.Frequency = metric.Frequency
Metrics[name] = existingMetric
}
}
} else {
Metrics[name] = metric
}
return nil
}

190
internal/memorystore/configSchema.go Normal file
View File

@@ -0,0 +1,190 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
const configSchema = `{
"type": "object",
"properties": {
"jobList": {
"description": "Job list defaults. Applies to user- and jobs views.",
"type": "object",
"properties": {
"usePaging": {
"description": "If classic paging is used instead of continuous scrolling by default.",
"type": "boolean"
},
"showFootprint": {
"description": "If footprint bars are shown as first column by default.",
"type": "boolean"
}
}
},
"nodeList": {
"description": "Node list defaults. Applies to node list view.",
"type": "object",
"properties": {
"usePaging": {
"description": "If classic paging is used instead of continuous scrolling by default.",
"type": "boolean"
}
}
},
"jobView": {
"description": "Job view defaults.",
"type": "object",
"properties": {
"showPolarPlot": {
"description": "If the job metric footprints polar plot is shown by default.",
"type": "boolean"
},
"showFootprint": {
"description": "If the annotated job metric footprint bars are shown by default.",
"type": "boolean"
},
"showRoofline": {
"description": "If the job roofline plot is shown by default.",
"type": "boolean"
},
"showStatTable": {
"description": "If the job metric statistics table is shown by default.",
"type": "boolean"
}
}
},
"metricConfig": {
"description": "Global initial metric selections for primary views of all clusters.",
"type": "object",
"properties": {
"jobListMetrics": {
"description": "Initial metrics shown for new users in job lists (User and jobs view).",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"jobViewPlotMetrics": {
"description": "Initial metrics shown for new users as job view metric plots.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"jobViewTableMetrics": {
"description": "Initial metrics shown for new users in job view statistics table.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"clusters": {
"description": "Overrides for global defaults by cluster and subcluster.",
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"description": "The name of the cluster."
},
"jobListMetrics": {
"description": "Initial metrics shown for new users in job lists (User and jobs view) for subcluster.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"jobViewPlotMetrics": {
"description": "Initial metrics shown for new users as job view timeplots for subcluster.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"jobViewTableMetrics": {
"description": "Initial metrics shown for new users in job view statistics table for subcluster.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"subClusters": {
"description": "The array of overrides per subcluster.",
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"description": "The name of the subcluster.",
"type": "string"
},
"jobListMetrics": {
"description": "Initial metrics shown for new users in job lists (User and jobs view) for subcluster.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"jobViewPlotMetrics": {
"description": "Initial metrics shown for new users as job view timeplots for subcluster.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
},
"jobViewTableMetrics": {
"description": "Initial metrics shown for new users in job view statistics table for subcluster.",
"type": "array",
"items": {
"type": "string",
"minItems": 1
}
}
},
"required": ["name"],
"minItems": 1
}
}
},
"required": ["name", "subClusters"],
"minItems": 1
}
}
}
},
"plotConfiguration": {
"description": "Initial settings for plot render options.",
"type": "object",
"properties": {
"colorBackground": {
"description": "If the metric plot backgrounds are initially colored by threshold limits.",
"type": "boolean"
},
"plotsPerRow": {
"description": "How many plots are initially rendered in per row. Applies to job, single node, and analysis views.",
"type": "integer"
},
"lineWidth": {
"description": "Initial thickness of rendered plotlines. Applies to metric plot, job compare plot and roofline.",
"type": "integer"
},
"colorScheme": {
"description": "Initial colorScheme to be used for metric plots.",
"type": "array",
"items": {
"type": "string"
}
}
}
}
}
}`

7
internal/memorystore/debug.go
View File

@@ -1,3 +1,8 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
@@ -41,7 +46,7 @@ func (l *Level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf [
depth += 1
objitems := 0
for name, mc := range m.Metrics {
if b := l.metrics[mc.Offset]; b != nil {
if b := l.metrics[mc.offset]; b != nil {
for i := 0; i < depth; i++ {
buf = append(buf, '\t')
}

16
internal/memorystore/healthcheck.go
View File

@@ -1,3 +1,8 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
@@ -6,29 +11,28 @@ import (
"time"
)
// This is a threshold that allows a node to be healthy with certain number of data points missing.
// MaxMissingDataPoints is a threshold that allows a node to be healthy with certain number of data points missing.
// Suppose a node does not receive last 5 data points, then healthCheck endpoint will still say a
// node is healthy. Anything more than 5 missing points in metrics of the node will deem the node unhealthy.
const MaxMissingDataPoints int64 = 5
// This is a threshold which allows upto certain number of metrics in a node to be unhealthly.
// MaxUnhealthyMetrics is a threshold which allows upto certain number of metrics in a node to be unhealthly.
// Works with MaxMissingDataPoints. Say 5 metrics (including submetrics) do not receive the last
// MaxMissingDataPoints data points, then the node will be deemed healthy. Any more metrics that does
// not receive data for MaxMissingDataPoints data points will deem the node unhealthy.
const MaxUnhealthyMetrics int64 = 5
func (b *buffer) healthCheck() int64 {
// Check if the buffer is empty
if b.data == nil {
return 1
}
buffer_end := b.start + b.frequency*int64(len(b.data))
bufferEnd := b.start + b.frequency*int64(len(b.data))
t := time.Now().Unix()
// Check if the buffer is too old
if t-buffer_end > MaxMissingDataPoints*b.frequency {
if t-bufferEnd > MaxMissingDataPoints*b.frequency {
return 1
}
@@ -40,7 +44,7 @@ func (l *Level) healthCheck(m *MemoryStore, count int64) (int64, error) {
defer l.lock.RUnlock()
for _, mc := range m.Metrics {
if b := l.metrics[mc.Offset]; b != nil {
if b := l.metrics[mc.offset]; b != nil {
count += b.healthCheck()
}
}

7
internal/memorystore/level.go
View File

@@ -1,3 +1,8 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
@@ -77,7 +82,7 @@ func (l *Level) free(t int64) (int, error) {
delme, m := b.free(t)
n += m
if delme {
if cap(b.data) == BUFFER_CAP {
if cap(b.data) == BufferCap {
bufferPool.Put(b)
}
l.metrics[i] = nil

24
internal/memorystore/lineprotocol.go
View File

@@ -1,3 +1,8 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
@@ -7,8 +12,6 @@ import (
"sync"
"time"
"github.com/ClusterCockpit/cc-backend/internal/avro"
"github.com/ClusterCockpit/cc-backend/internal/config"
"github.com/ClusterCockpit/cc-lib/schema"
"github.com/influxdata/line-protocol/v2/lineprotocol"
"github.com/nats-io/nats.go"
@@ -77,10 +80,10 @@ import (
// return nil
// }
// Connect to a nats server and subscribe to "updates". This is a blocking
// function. handleLine will be called for each line recieved via nats.
// Send `true` through the done channel for gracefull termination.
func ReceiveNats(conf *(config.NatsConfig),
// ReceiveNats connects to a nats server and subscribes to "updates". This is a
// blocking function. handleLine will be called for each line recieved via
// nats. Send `true` through the done channel for gracefull termination.
func ReceiveNats(conf *(NatsConfig),
ms *MemoryStore,
workers int,
ctx context.Context,
@@ -170,7 +173,7 @@ func reorder(buf, prefix []byte) []byte {
for i := m - 1; i >= 0; i-- {
buf[i+n] = buf[i]
}
for i := 0; i < n; i++ {
for i := range n {
buf[i] = prefix[i]
}
return buf
@@ -329,14 +332,15 @@ func decodeLine(dec *lineprotocol.Decoder,
time := t.Unix()
if config.MetricStoreKeys.Checkpoints.FileFormat != "json" {
avro.LineProtocolMessages <- &avro.AvroStruct{
if Keys.Checkpoints.FileFormat != "json" {
LineProtocolMessages <- &AvroStruct{
MetricName: string(metricBuf),
Cluster: cluster,
Node: host,
Selector: append([]string{}, selector...),
Value: metric.Value,
Timestamp: time}
Timestamp: time,
}
}
if err := ms.WriteToLevel(lvl, selector, time, []Metric{metric}); err != nil {

70
internal/memorystore/memorystore.go
View File

@@ -1,3 +1,8 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
@@ -11,8 +16,6 @@ import (
"syscall"
"time"
"github.com/ClusterCockpit/cc-backend/internal/avro"
"github.com/ClusterCockpit/cc-backend/internal/config"
"github.com/ClusterCockpit/cc-lib/resampler"
"github.com/ClusterCockpit/cc-lib/runtimeEnv"
"github.com/ClusterCockpit/cc-lib/schema"
@@ -30,39 +33,36 @@ var NumWorkers int = 4
func init() {
maxWorkers := 10
NumWorkers = runtime.NumCPU()/2 + 1
if NumWorkers > maxWorkers {
NumWorkers = maxWorkers
}
NumWorkers = min(runtime.NumCPU()/2+1, maxWorkers)
}
type Metric struct {
Name string
Value schema.Float
MetricConfig config.MetricConfig
MetricConfig MetricConfig
}
type MemoryStore struct {
Metrics map[string]config.MetricConfig
Metrics map[string]MetricConfig
root Level
}
func Init(wg *sync.WaitGroup) {
startupTime := time.Now()
//Pass the config.MetricStoreKeys
InitMetrics(config.Metrics)
// Pass the config.MetricStoreKeys
InitMetrics(Metrics)
ms := GetMemoryStore()
d, err := time.ParseDuration(config.MetricStoreKeys.Checkpoints.Restore)
d, err := time.ParseDuration(Keys.Checkpoints.Restore)
if err != nil {
log.Fatal(err)
}
restoreFrom := startupTime.Add(-d)
log.Printf("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
files, err := ms.FromCheckpointFiles(config.MetricStoreKeys.Checkpoints.RootDir, restoreFrom.Unix())
files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
if err != nil {
log.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
@@ -85,7 +85,7 @@ func Init(wg *sync.WaitGroup) {
Retention(wg, ctx)
Checkpointing(wg, ctx)
Archiving(wg, ctx)
avro.DataStaging(wg, ctx)
DataStaging(wg, ctx)
wg.Add(1)
sigs := make(chan os.Signal, 1)
@@ -97,8 +97,8 @@ func Init(wg *sync.WaitGroup) {
shutdown()
}()
if config.MetricStoreKeys.Nats != nil {
for _, natsConf := range config.MetricStoreKeys.Nats {
if Keys.Nats != nil {
for _, natsConf := range Keys.Nats {
// TODO: When multiple nats configs share a URL, do a single connect.
wg.Add(1)
nc := natsConf
@@ -114,9 +114,9 @@ func Init(wg *sync.WaitGroup) {
}
}
// Create a new, initialized instance of a MemoryStore.
// InitMetrics creates a new, initialized instance of a MemoryStore.
// Will panic if values in the metric configurations are invalid.
func InitMetrics(metrics map[string]config.MetricConfig) {
func InitMetrics(metrics map[string]MetricConfig) {
singleton.Do(func() {
offset := 0
for key, cfg := range metrics {
@@ -124,10 +124,10 @@ func InitMetrics(metrics map[string]config.MetricConfig) {
panic("[METRICSTORE]> invalid frequency")
}
metrics[key] = config.MetricConfig{
metrics[key] = MetricConfig{
Frequency: cfg.Frequency,
Aggregation: cfg.Aggregation,
Offset: offset,
offset: offset,
}
offset += 1
}
@@ -151,17 +151,17 @@ func GetMemoryStore() *MemoryStore {
}
func Shutdown() {
log.Printf("[METRICSTORE]> Writing to '%s'...\n", config.MetricStoreKeys.Checkpoints.RootDir)
log.Printf("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
var files int
var err error
ms := GetMemoryStore()
if config.MetricStoreKeys.Checkpoints.FileFormat == "json" {
files, err = ms.ToCheckpoint(config.MetricStoreKeys.Checkpoints.RootDir, lastCheckpoint.Unix(), time.Now().Unix())
if Keys.Checkpoints.FileFormat == "json" {
files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, lastCheckpoint.Unix(), time.Now().Unix())
} else {
files, err = avro.GetAvroStore().ToCheckpoint(config.MetricStoreKeys.Checkpoints.RootDir, true)
close(avro.LineProtocolMessages)
files, err = GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, true)
close(LineProtocolMessages)
}
if err != nil {
@@ -234,7 +234,7 @@ func Shutdown() {
func getName(m *MemoryStore, i int) string {
for key, val := range m.Metrics {
if val.Offset == i {
if val.offset == i {
return key
}
}
@@ -246,7 +246,7 @@ func Retention(wg *sync.WaitGroup, ctx context.Context) {
go func() {
defer wg.Done()
d, err := time.ParseDuration(config.MetricStoreKeys.RetentionInMemory)
d, err := time.ParseDuration(Keys.RetentionInMemory)
if err != nil {
log.Fatal(err)
}
@@ -311,11 +311,11 @@ func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metric
continue
}
b := l.metrics[metric.MetricConfig.Offset]
b := l.metrics[metric.MetricConfig.offset]
if b == nil {
// First write to this metric and level
b = newBuffer(ts, metric.MetricConfig.Frequency)
l.metrics[metric.MetricConfig.Offset] = b
l.metrics[metric.MetricConfig.offset] = b
}
nb, err := b.write(ts, metric.Value)
@@ -325,7 +325,7 @@ func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metric
// Last write created a new buffer...
if b != nb {
l.metrics[metric.MetricConfig.Offset] = nb
l.metrics[metric.MetricConfig.offset] = nb
}
}
return nil
@@ -337,17 +337,17 @@ func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metric
// the range asked for if no data was available.
func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
if from > to {
return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range\n")
return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
}
minfo, ok := m.Metrics[metric]
if !ok {
return nil, 0, 0, 0, errors.New("[METRICSTORE]> unkown metric: \n" + metric)
return nil, 0, 0, 0, errors.New("[METRICSTORE]> unkown metric: " + metric)
}
n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
err := m.root.findBuffers(selector, minfo.Offset, func(b *buffer) error {
err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
cdata, cfrom, cto, err := b.read(from, to, data)
if err != nil {
return err
@@ -381,14 +381,14 @@ func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, reso
if err != nil {
return nil, 0, 0, 0, err
} else if n == 0 {
return nil, 0, 0, 0, errors.New("[METRICSTORE]> metric or host not found\n")
return nil, 0, 0, 0, errors.New("[METRICSTORE]> metric or host not found")
} else if n > 1 {
if minfo.Aggregation == config.AvgAggregation {
if minfo.Aggregation == AvgAggregation {
normalize := 1. / schema.Float(n)
for i := 0; i < len(data); i++ {
data[i] *= normalize
}
} else if minfo.Aggregation != config.SumAggregation {
} else if minfo.Aggregation != SumAggregation {
return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
}
}

12
internal/memorystore/stats.go
View File

@@ -1,10 +1,14 @@
// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package memorystore
import (
"errors"
"math"
"github.com/ClusterCockpit/cc-backend/internal/config"
"github.com/ClusterCockpit/cc-lib/util"
)
@@ -78,7 +82,7 @@ func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int6
n, samples := 0, 0
avg, min, max := util.Float(0), math.MaxFloat32, -math.MaxFloat32
err := m.root.findBuffers(selector, minfo.Offset, func(b *buffer) error {
err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
stats, cfrom, cto, err := b.stats(from, to)
if err != nil {
return err
@@ -105,9 +109,9 @@ func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int6
return nil, 0, 0, ErrNoData
}
if minfo.Aggregation == config.AvgAggregation {
if minfo.Aggregation == AvgAggregation {
avg /= util.Float(n)
} else if n > 1 && minfo.Aggregation != config.SumAggregation {
} else if n > 1 && minfo.Aggregation != SumAggregation {
return nil, 0, 0, errors.New("invalid aggregation")
}