ClusterCockpit/cc-metric-store
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-metric-store.git synced 2024-12-27 09:19:06 +01:00
Code Issues Packages Projects Releases Wiki Activity

add config flag; fix bug in archive

Browse Source
This commit is contained in:
Lou Knauer 2021-12-15 10:58:03 +01:00
parent ac7e981321
commit 50731e43a8
2 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
archive.go
View File

@ -8,6 +8,7 @@ import (
"fmt"
"io"
"io/fs"
"log"
"os"
"path"
"path/filepath"
@ -27,6 +28,7 @@ type CheckpointMetrics struct {
type CheckpointFile struct {
From int64 `json:"from"`
To int64 `json:"to"`
Metrics map[string]*CheckpointMetrics `json:"metrics"`
Children map[string]*CheckpointFile `json:"children"`
}
@ -75,6 +77,7 @@ func (l *level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFil
retval := &CheckpointFile{
From: from,
To: to,
Metrics: make(map[string]*CheckpointMetrics),
Children: make(map[string]*CheckpointFile),
}
@ -168,8 +171,8 @@ func (l *level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
// This function can only be called once and before the very first write or read.
// Different host's data is loaded to memory in parallel.
func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
workers := runtime.NumCPU() / 2
work := make(chan [2]string, workers)
workers := (runtime.NumCPU() / 2) + 1
work := make(chan [2]string, workers*2)
errs := make(chan error, 1)
var wg sync.WaitGroup
wg.Add(workers + 1)
@ -216,8 +219,7 @@ func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
lvl := m.root.findLevelOrCreate(host[:], len(m.metrics))
n, err := lvl.fromCheckpoint(filepath.Join(dir, host[0], host[1]), from, m)
if err != nil {
errs <- err
return
log.Fatalf("error while loading checkpoints: %s", err.Error())
}
atomic.AddInt32(&loadedFiles, int32(n))
}
@ -249,7 +251,8 @@ func (l *level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
minfo, ok := m.metrics[name]
if !ok {
return errors.New("Unkown metric: " + name)
continue
// return errors.New("Unkown metric: " + name)
}
prev := l.metrics[minfo.offset]
@ -338,6 +341,10 @@ func (l *level) fromCheckpoint(dir string, from int64, m *MemoryStore) (int, err
return filesLoaded, err
}
if cf.To != 0 && cf.To < from {
continue
}
if err = l.loadFile(cf, m); err != nil {
return filesLoaded, err
}

7
metric-store.go
View File

@ -4,6 +4,7 @@ import (
"bufio"
"context"
"encoding/json"
"flag"
"fmt"
"log"
"os"
@ -212,8 +213,12 @@ func intervals(wg *sync.WaitGroup, ctx context.Context) {
}
func main() {
var configFile string
flag.StringVar(&configFile, "config", "./config.json", "configuration file")
flag.Parse()
startupTime := time.Now()
conf = loadConfiguration("config.json")
conf = loadConfiguration(configFile)
memoryStore = NewMemoryStore(conf.Metrics)
restoreFrom := startupTime.Add(-time.Duration(conf.Checkpoints.Restore) * time.Second)