The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go has been updated successfully.

	wg.Go(func() {

		d := 12 * time.Hour // default checkpoint interval
		if Keys.CheckpointInterval != "" {
			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
			if err != nil {
				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
			} else {
				d = parsed
			}
		}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements checkpoint persistence for the in-memory metric store.
//
// Checkpoints enable graceful restarts by periodically saving in-memory metric
// data to disk in JSON or binary format. The checkpoint system:
//
// Key Features:
//   - Periodic background checkpointing via the Checkpointing() worker
//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
//   - Parallel checkpoint creation and loading using worker pools
//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
//   - Only saves unarchived data (archived data is already persisted elsewhere)
//   - GC optimization during loading to prevent excessive heap growth
//
// Checkpoint Workflow:
//  1. Init() loads checkpoints within retention window at startup
//  2. Checkpointing() worker periodically saves new data
//  3. Shutdown() writes final checkpoint before exit
//
// File Organization:
//
//	checkpoints/
//	  cluster1/
//	    host001/
//	      1234567890.json  (JSON format: full subtree snapshot)
//	      1234567890.bin   (binary format: full subtree snapshot)
//	      current.wal      (WAL format: append-only per-entry log)
//	    host002/
//	      ...
package metricstore

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

const (
	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
)

// CheckpointMetrics represents metric data in a checkpoint file.
// Whenever the structure changes, update MarshalJSON as well!
type CheckpointMetrics struct {
	Data      []schema.Float `json:"data"`
	Frequency int64          `json:"frequency"`
	Start     int64          `json:"start"`
}

// CheckpointFile represents the hierarchical structure of a checkpoint file.
// It mirrors the Level tree structure from the MemoryStore.
type CheckpointFile struct {
	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
	Children map[string]*CheckpointFile    `json:"children"`
	From     int64                         `json:"from"`
	To       int64                         `json:"to"`
}

// lastCheckpoint tracks the timestamp of the last checkpoint creation.
var (
	lastCheckpoint   time.Time
	lastCheckpointMu sync.Mutex
)

// Checkpointing starts a background worker that periodically saves metric data to disk.
//
// Checkpoints are written every 12 hours (hardcoded).
//
// Format behaviour:
//   - "json": Periodic checkpointing every checkpointInterval
//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
	lastCheckpointMu.Lock()
	lastCheckpoint = time.Now()
	lastCheckpointMu.Unlock()

	ms := GetMemoryStore()

	wg.Go(func() {

		const checkpointInterval = 12 * time.Hour
		d := checkpointInterval

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				lastCheckpointMu.Lock()
				from := lastCheckpoint
				lastCheckpointMu.Unlock()

				now := time.Now()
				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))

				if Keys.Checkpoints.FileFormat == "wal" {
					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
					} else {
						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
						// Rotate WAL files for successfully checkpointed hosts.
						RotateWALFiles(hostDirs)
					}
				} else {
					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
					}
				}
			}
		}
	})
}

// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
//
// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
// This method manually constructs JSON to avoid allocations and interface conversions.
func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
	buf := make([]byte, 0, 128+len(cm.Data)*8)
	buf = append(buf, `{"frequency":`...)
	buf = strconv.AppendInt(buf, cm.Frequency, 10)
	buf = append(buf, `,"start":`...)
	buf = strconv.AppendInt(buf, cm.Start, 10)
	buf = append(buf, `,"data":[`...)
	for i, x := range cm.Data {
		if i != 0 {
			buf = append(buf, ',')
		}
		if x.IsNaN() {
			buf = append(buf, `null`...)
		} else {
			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
		}
	}
	buf = append(buf, `]}`...)
	return buf, nil
}

// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format).
//
// Metrics at root and cluster levels are skipped. One file per host is created.
// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
// at a time, allowing concurrent writes/reads to other hosts.
//
// Returns the number of checkpoint files created and any errors encountered.
func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
	// Pre-calculate capacity by counting cluster/host pairs
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		dir      string
		selector []string
	}

	n, errs := int32(0), int32(0)

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()

			for workItem := range work {
				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
					if err == ErrNoNewArchiveData {
						continue
					}

					cclog.Errorf("[METRICSTORE]> error while checkpointing %#v: %s", workItem.selector, err.Error())
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
				}
			}
		}()
	}

	for i := 0; i < len(levels); i++ {
		dir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			dir:      dir,
			selector: selectors[i],
		}
	}

	close(work)
	wg.Wait()

	if errs > 0 {
		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
	}
	return int(n), nil
}

// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
// Skips metrics that are already archived. Returns nil if no unarchived data exists.
func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	retval := &CheckpointFile{
		From:     from,
		To:       to,
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	for metric, minfo := range m.Metrics {
		b := l.metrics[minfo.offset]
		if b == nil {
			continue
		}

		allArchived := true
		b.iterFromTo(from, to, func(b *buffer) error {
			if !b.archived {
				allArchived = false
				return fmt.Errorf("stop") // Early termination signal
			}
			return nil
		})

		if allArchived {
			continue
		}

		data := make([]schema.Float, (to-from)/b.frequency+1)
		data, start, end, err := b.read(from, to, data)
		if err != nil {
			return nil, err
		}

		for i := int((end - start) / b.frequency); i < len(data); i++ {
			data[i] = schema.NaN
		}

		retval.Metrics[metric] = &CheckpointMetrics{
			Frequency: b.frequency,
			Start:     start,
			Data:      data,
		}
	}

	for name, child := range l.children {
		val, err := child.toCheckpointFile(from, to, m)
		if err != nil {
			return nil, err
		}

		if val != nil {
			retval.Children[name] = val
		}
	}

	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
		return nil, nil
	}

	return retval, nil
}

// toCheckpoint writes a Level's data to a JSON checkpoint file.
// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.
func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}

	if cf == nil {
		return ErrNoNewArchiveData
	}

	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil && os.IsNotExist(err) {
		err = os.MkdirAll(dir, CheckpointDirPerms)
		if err == nil {
			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
		}
	}
	if err != nil {
		return err
	}
	defer f.Close()

	bw := bufio.NewWriter(f)
	if err = json.NewEncoder(bw).Encode(cf); err != nil {
		return err
	}

	return bw.Flush()
}

// enqueueCheckpointHosts traverses checkpoint directory and enqueues cluster/host pairs.
// Returns error if directory structure is invalid.
func enqueueCheckpointHosts(dir string, work chan<- [2]string) error {
	clustersDir, err := os.ReadDir(dir)
	if err != nil {
		return err
	}

	for _, clusterDir := range clustersDir {
		if !clusterDir.IsDir() {
			return errors.New("[METRICSTORE]> expected only directories at first level of checkpoints/ directory")
		}

		hostsDir, err := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
		if err != nil {
			return err
		}

		for _, hostDir := range hostsDir {
			if !hostDir.IsDir() {
				return errors.New("[METRICSTORE]> expected only directories at second level of checkpoints/ directory")
			}

			work <- [2]string{clusterDir.Name(), hostDir.Name()}
		}
	}

	return nil
}

// FromCheckpoint loads checkpoint files from disk into memory in parallel.
//
// Uses worker pool to load cluster/host combinations. Returns number of files
// loaded and any errors.
func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
	var wg sync.WaitGroup
	work := make(chan [2]string, Keys.NumWorkers*4)
	n, errs := int32(0), int32(0)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for host := range work {
				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
					atomic.AddInt32(&errs, 1)
				}
				atomic.AddInt32(&n, int32(nn))
			}
		}()
	}

	err := enqueueCheckpointHosts(dir, work)
	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}

	if errs > 0 {
		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
	}
	return int(n), nil
}

// FromCheckpointFiles is the main entry point for loading checkpoints at startup.
//
// Creates checkpoint directory if it doesn't exist. This function must be called
// before any writes or reads, and can only be called once.
func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, CheckpointDirPerms)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
		}
		cclog.Debugf("[METRICSTORE]> %#v Directory created successfully", dir)
	}

	return m.FromCheckpoint(dir, from)
}

func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)
	cf := &CheckpointFile{}
	if err := json.NewDecoder(br).Decode(cf); err != nil {
		return err
	}

	if cf.To != 0 && cf.To < from {
		return nil
	}

	if err := l.loadFile(cf, m); err != nil {
		return err
	}

	return nil
}

func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
	for name, metric := range cf.Metrics {
		n := len(metric.Data)
		b := &buffer{
			frequency: metric.Frequency,
			start:     metric.Start,
			data:      metric.Data[0:n:n],
			prev:      nil,
			next:      nil,
			archived:  true,
		}

		minfo, ok := m.Metrics[name]
		if !ok {
			continue
		}

		prev := l.metrics[minfo.offset]
		if prev == nil {
			l.metrics[minfo.offset] = b
		} else {
			if prev.start > b.start {
				return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start)
			}

			b.prev = prev
			prev.next = b
		}
		l.metrics[minfo.offset] = b
	}

	if len(cf.Children) > 0 && l.children == nil {
		l.children = make(map[string]*Level)
	}

	for sel, childCf := range cf.Children {
		child, ok := l.children[sel]
		if !ok {
			child = &Level{
				metrics:  make([]*buffer, len(m.Metrics)),
				children: nil,
			}
			l.children[sel] = child
		}

		if err := child.loadFile(childCf, m); err != nil {
			return err
		}
	}

	return nil
}

// fromCheckpoint loads all checkpoint files (JSON, binary snapshot, WAL) for a
// single host directory. Snapshot files are loaded first (sorted by timestamp),
// then current.wal is replayed on top.
func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {
	direntries, err := os.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return 0, nil
		}
		return 0, err
	}

	allFiles := make([]fs.DirEntry, 0)
	var walEntry fs.DirEntry
	filesLoaded := 0

	for _, e := range direntries {
		if e.IsDir() {
			// Legacy: skip subdirectories (only used by old Avro format).
			// These are ignored; their data is not loaded.
			cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir)
			continue
		}

		name := e.Name()
		if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") {
			allFiles = append(allFiles, e)
		} else if name == "current.wal" {
			walEntry = e
		}
		// Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes).
	}

	files, err := findFiles(allFiles, from, true)
	if err != nil {
		return filesLoaded, err
	}

	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
		".json": l.loadJSONFile,
		".bin":  l.loadBinaryFile,
	}

	for _, filename := range files {
		ext := filepath.Ext(filename)
		loader := loaders[ext]
		if loader == nil {
			cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename)
			continue
		}

		err := func() error {
			f, err := os.Open(path.Join(dir, filename))
			if err != nil {
				return err
			}
			defer f.Close()
			return loader(m, f, from)
		}()
		if err != nil {
			return filesLoaded, err
		}
		filesLoaded++
	}

	// Replay WAL after all snapshot files so it fills in data since the last snapshot.
	if walEntry != nil {
		err := func() error {
			f, err := os.Open(path.Join(dir, walEntry.Name()))
			if err != nil {
				return err
			}
			defer f.Close()
			return l.loadWALFile(m, f, from)
		}()
		if err != nil {
			// WAL errors are non-fatal: the snapshot already loaded the bulk of data.
			cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err)
		} else {
			filesLoaded++
		}
	}

	return filesLoaded, nil
}

// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename.
// Supports ".json" (format: "<ts>.json") and ".bin" (format: "<ts>.bin").
func parseTimestampFromFilename(name string) (int64, error) {
	switch {
	case strings.HasSuffix(name, ".json"):
		return strconv.ParseInt(name[:len(name)-5], 10, 64)
	case strings.HasSuffix(name, ".bin"):
		return strconv.ParseInt(name[:len(name)-4], 10, 64)
	default:
		return 0, fmt.Errorf("unknown checkpoint extension for file %q", name)
	}
}

// findFiles returns filenames from direntries whose timestamps satisfy the filter.
// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the
// last file before t if t falls between two files).
func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {
	nums := map[string]int64{}
	for _, e := range direntries {
		name := e.Name()
		if !strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".bin") {
			continue
		}

		ts, err := parseTimestampFromFilename(name)
		if err != nil {
			return nil, err
		}
		nums[name] = ts
	}

	sort.Slice(direntries, func(i, j int) bool {
		a, b := direntries[i], direntries[j]
		return nums[a.Name()] < nums[b.Name()]
	})

	if len(nums) == 0 {
		return nil, nil
	}

	filenames := make([]string, 0)

	for i, e := range direntries {
		ts1 := nums[e.Name()]

		if findMoreRecentFiles && t <= ts1 {
			filenames = append(filenames, e.Name())
		} else if !findMoreRecentFiles && ts1 <= t && ts1 != 0 {
			filenames = append(filenames, e.Name())
		}

		if i == len(direntries)-1 {
			continue
		}

		enext := direntries[i+1]
		ts2 := nums[enext.Name()]

		if findMoreRecentFiles {
			if ts1 < t && t < ts2 {
				filenames = append(filenames, e.Name())
			}
		}
	}

	return filenames, nil
}

-    const checkpointInterval = 12 * time.Hour

+    d := 12 * time.Hour // default checkpoint interval

+      parsed, err := time.ParseDuration(Keys.CheckpointInterval)

+        cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)


Now let me also update the `free()` method to simply delegate to `freeAndCheckEmpty` to avoid code duplication, since `free()` is the public-facing method used by `MemoryStore.Free()`.


Edit

func (l *Level) free(t int64) (int, error) {
	l.lock.Lock()
	defer l.lock.Unlock()

	n := 0
	for i, b := range l.metrics {
		if b != nil {
			delme, m := b.free(t)
			n += m
			if delme {
				if cap(b.data) != BufferCap {
					b.data = make([]schema.Float, 0, BufferCap)
				}
				b.lastUsed = time.Now().Unix()
				bufferPool.Put(b)
				l.metrics[i] = nil
			}
		}
	}

	for key, child := range l.children {
		m, empty, err := child.freeAndCheckEmpty(t)
		n += m
		if err != nil {
			return n, err
		}
		if empty {
			delete(l.children, key)
		}
	}

	return n, nil
}

func (l *Level) free(t int64) (int, error) {
	n, _, err := l.freeAndCheckEmpty(t)
	return n, err
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/level.go has been updated successfully.

func (l *Level) free(t int64) (int, error) {
	n, _, err := l.freeAndCheckEmpty(t)
	return n, err
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides level.go: Hierarchical tree structure for metric storage.
//
// # Level Architecture
//
// The Level type forms a tree structure where each node represents a level in the
// ClusterCockpit hierarchy: cluster → host → socket → core → hwthread, with special
// nodes for memory domains and accelerators.
//
// Structure:
//
//	Root Level (cluster="emmy")
//	├─ Level (host="node001")
//	│  ├─ Level (socket="0")
//	│  │  ├─ Level (core="0") [stores cpu0 metrics]
//	│  │  └─ Level (core="1") [stores cpu1 metrics]
//	│  └─ Level (socket="1")
//	│     └─ ...
//	└─ Level (host="node002")
//	   └─ ...
//
// Each Level can:
//   - Hold data (metrics slice of buffer pointers)
//   - Have child nodes (children map[string]*Level)
//   - Both simultaneously (inner nodes can store aggregated metrics)
//
// # Selector Paths
//
// Selectors are hierarchical paths: []string{"cluster", "host", "component"}.
// Example: []string{"emmy", "node001", "cpu0"} navigates to the cpu0 core level.
//
// # Concurrency
//
// RWMutex protects children map and metrics slice. Read-heavy workload (metric reads)
// uses RLock. Writes (new levels, buffer updates) use Lock. Double-checked locking
// prevents races during level creation.
package metricstore

import (
	"sync"
	"time"
	"unsafe"

	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-lib/v2/util"
)

// Level represents a node in the hierarchical metric storage tree.
//
// Can be both a leaf or inner node. Inner nodes hold data in 'metrics' for aggregated
// values (e.g., socket-level metrics derived from core-level data). Named "Level"
// instead of "node" to avoid confusion with cluster nodes (hosts).
//
// Fields:
//   - children: Map of child level names to Level pointers (e.g., "cpu0" → Level)
//   - metrics:  Slice of buffer pointers (one per metric, indexed by MetricConfig.offset)
//   - lock:     RWMutex for concurrent access (read-heavy, write-rare)
type Level struct {
	children map[string]*Level
	metrics  []*buffer
	lock     sync.RWMutex
}

// findLevelOrCreate navigates to or creates the level specified by selector.
//
// Recursively descends the tree, creating missing levels as needed. Uses double-checked
// locking: RLock first (fast path), then Lock if creation needed (slow path), then
// re-check after acquiring Lock to handle races.
//
// Example selector: []string{"emmy", "node001", "cpu0"}
// Navigates: root → emmy → node001 → cpu0, creating levels as needed.
//
// Parameters:
//   - selector: Hierarchical path (consumed recursively, decreasing depth)
//   - nMetrics: Number of metric slots to allocate in new levels
//
// Returns:
//   - *Level: The target level (existing or newly created)
//
// Note: sync.Map may improve performance for high-concurrency writes, but current
// approach suffices for read-heavy workload.
func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
	if len(selector) == 0 {
		return l
	}

	// Allow concurrent reads:
	l.lock.RLock()
	var child *Level
	var ok bool
	if l.children == nil {
		// Children map needs to be created...
		l.lock.RUnlock()
	} else {
		child, ok = l.children[selector[0]]
		l.lock.RUnlock()
		if ok {
			return child.findLevelOrCreate(selector[1:], nMetrics)
		}
	}

	// The level does not exist, take write lock for unique access:
	l.lock.Lock()
	// While this thread waited for the write lock, another thread
	// could have created the child node.
	if l.children != nil {
		child, ok = l.children[selector[0]]
		if ok {
			l.lock.Unlock()
			return child.findLevelOrCreate(selector[1:], nMetrics)
		}
	}

	child = &Level{
		metrics:  make([]*buffer, nMetrics),
		children: nil,
	}

	if l.children != nil {
		l.children[selector[0]] = child
	} else {
		l.children = map[string]*Level{selector[0]: child}
	}
	l.lock.Unlock()
	return child.findLevelOrCreate(selector[1:], nMetrics)
}

// collectPaths gathers all selector paths at the specified depth in the tree.
//
// Recursively traverses children, collecting paths when currentDepth+1 == targetDepth.
// Each path is a selector that can be used with findLevel() or findBuffers().
//
// Explicitly copies slices to avoid shared underlying arrays between siblings, preventing
// unintended mutations.
//
// Parameters:
//   - currentDepth: Depth of current level (0 = root)
//   - targetDepth:  Depth to collect paths from
//   - currentPath:  Path accumulated so far
//   - results:      Output slice (appended to)
//
// Example: collectPaths(0, 2, []string{}, &results) collects all 2-level paths
// like []string{"emmy", "node001"}, []string{"emmy", "node002"}, etc.
func (l *Level) collectPaths(currentDepth, targetDepth int, currentPath []string, results *[][]string) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	for key, child := range l.children {
		if child == nil {
			continue
		}

		// We explicitly make a new slice and copy data to avoid sharing underlying arrays between siblings
		newPath := make([]string, len(currentPath))
		copy(newPath, currentPath)
		newPath = append(newPath, key)

		// Check depth, and just return if depth reached
		if currentDepth+1 == targetDepth {
			*results = append(*results, newPath)
		} else {
			child.collectPaths(currentDepth+1, targetDepth, newPath, results)
		}
	}
}

// free removes buffers older than the retention threshold from the entire subtree.
//
// Recursively frees buffers in this level's metrics and all child levels. Buffers
// with standard capacity (BufferCap) are returned to the pool. Called by the
// retention worker to enforce retention policies.
//
// Parameters:
//   - t: Retention threshold timestamp (Unix seconds)
//
// Returns:
//   - int:   Total number of buffers freed in this subtree
//   - error: Non-nil on failure (propagated from children)
func (l *Level) free(t int64) (int, error) {
	l.lock.Lock()
	defer l.lock.Unlock()

	n := 0
	for i, b := range l.metrics {
		if b != nil {
			delme, m := b.free(t)
			n += m
			if delme {
				if cap(b.data) != BufferCap {
					b.data = make([]schema.Float, 0, BufferCap)
				}
				b.lastUsed = time.Now().Unix()
				bufferPool.Put(b)
				l.metrics[i] = nil
			}
		}
	}

	for key, child := range l.children {
		m, empty, err := child.freeAndCheckEmpty(t)
		n += m
		if err != nil {
			return n, err
		}
		if empty {
			delete(l.children, key)
		}
	}

	return n, nil
}

// freeAndCheckEmpty performs free() and atomically checks if the level is empty
// while still holding its own lock, avoiding a TOCTOU race between free() and
// a separate isEmpty() call.
func (l *Level) freeAndCheckEmpty(t int64) (int, bool, error) {
	l.lock.Lock()
	defer l.lock.Unlock()

	n := 0
	for i, b := range l.metrics {
		if b != nil {
			delme, m := b.free(t)
			n += m
			if delme {
				if cap(b.data) != BufferCap {
					b.data = make([]schema.Float, 0, BufferCap)
				}
				b.lastUsed = time.Now().Unix()
				bufferPool.Put(b)
				l.metrics[i] = nil
			}
		}
	}

	for key, child := range l.children {
		m, empty, err := child.freeAndCheckEmpty(t)
		n += m
		if err != nil {
			return n, false, err
		}
		if empty {
			delete(l.children, key)
		}
	}

	// Check emptiness while still holding the lock
	empty := len(l.children) == 0
	if empty {
		for _, b := range l.metrics {
			if b != nil {
				empty = false
				break
			}
		}
	}

	return n, empty, nil
}

// isEmpty returns true if this level has no metrics and no children.
func (l *Level) isEmpty() bool {
	l.lock.RLock()
	defer l.lock.RUnlock()

	for _, b := range l.metrics {
		if b != nil {
			return false
		}
	}
	return len(l.children) == 0
}

// forceFree removes the oldest buffer from each metric chain in the subtree.
//
// Unlike free(), which removes based on time threshold, this unconditionally removes
// the oldest buffer in each chain. Used by MemoryUsageTracker when memory cap is
// exceeded and time-based retention is insufficient.
//
// Recursively processes current level's metrics and all child levels.
//
// Returns:
//   - int:   Total number of buffers freed in this subtree
//   - error: Non-nil on failure (propagated from children)
func (l *Level) forceFree() (int, error) {
	l.lock.Lock()
	defer l.lock.Unlock()

	n := 0

	// Iterate over metrics in the current level
	for i, b := range l.metrics {
		if b != nil {
			// Attempt to free the oldest buffer in this chain
			delme, freedCount := b.forceFreeOldest()
			n += freedCount

			// If delme is true, it means 'b' itself (the head) was the oldest
			// and needs to be removed from the slice.
			if delme {
				b.next = nil
				b.prev = nil
				if cap(b.data) != BufferCap {
					b.data = make([]schema.Float, 0, BufferCap)
				}
				b.lastUsed = time.Now().Unix()
				bufferPool.Put(b)
				l.metrics[i] = nil
			}
		}
	}

	// Recursively traverse children
	for _, child := range l.children {
		m, err := child.forceFree()
		n += m
		if err != nil {
			return n, err
		}
	}

	return n, nil
}

// sizeInBytes calculates the total memory usage of all buffers in the subtree.
//
// Recursively sums buffer data sizes (count of Float values × sizeof(Float)) across
// this level's metrics and all child levels. Used by MemoryUsageTracker to enforce
// memory cap limits.
//
// Returns:
//   - int64: Total bytes used by buffer data in this subtree
func (l *Level) sizeInBytes() int64 {
	l.lock.RLock()
	defer l.lock.RUnlock()
	size := int64(0)

	for _, b := range l.metrics {
		if b != nil {
			size += b.count() * int64(unsafe.Sizeof(schema.Float(0)))
			size += b.bufferCount() * int64(unsafe.Sizeof(buffer{}))
		}
	}

	for _, child := range l.children {
		size += child.sizeInBytes()
	}

	return size
}

// findLevel navigates to the level specified by selector, returning nil if not found.
//
// Read-only variant of findLevelOrCreate. Does not create missing levels.
// Recursively descends the tree following the selector path.
//
// Parameters:
//   - selector: Hierarchical path (e.g., []string{"emmy", "node001", "cpu0"})
//
// Returns:
//   - *Level: The target level, or nil if any component in the path does not exist
func (l *Level) findLevel(selector []string) *Level {
	if len(selector) == 0 {
		return l
	}

	l.lock.RLock()
	defer l.lock.RUnlock()

	lvl := l.children[selector[0]]
	if lvl == nil {
		return nil
	}

	return lvl.findLevel(selector[1:])
}

// findBuffers invokes callback on all buffers matching the selector pattern.
//
// Supports flexible selector patterns (from cc-lib/util.Selector):
//   - Exact match: Selector element with String set (e.g., "node001")
//   - Group match: Selector element with Group set (e.g., ["cpu0", "cpu2", "cpu4"])
//   - Wildcard:    Selector element with Any=true (matches all children)
//
// Empty selector (len==0) matches current level's buffer at 'offset' and recursively
// all descendant buffers at the same offset (used for aggregation queries).
//
// Parameters:
//   - selector: Pattern to match (consumed recursively)
//   - offset:   Metric index in metrics slice (from MetricConfig.offset)
//   - f:        Callback invoked on each matching buffer
//
// Returns:
//   - error: First error returned by callback, or nil if all succeeded
//
// Example:
//
//	// Find all cpu0 buffers across all hosts:
//	findBuffers([]Selector{{Any: true}, {String: "cpu0"}}, metricOffset, callback)
func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
	l.lock.RLock()
	defer l.lock.RUnlock()

	if len(selector) == 0 {
		b := l.metrics[offset]
		if b != nil {
			return f(b)
		}

		for _, lvl := range l.children {
			err := lvl.findBuffers(nil, offset, f)
			if err != nil {
				return err
			}
		}
		return nil
	}

	sel := selector[0]
	if len(sel.String) != 0 && l.children != nil {
		lvl, ok := l.children[sel.String]
		if ok {
			err := lvl.findBuffers(selector[1:], offset, f)
			if err != nil {
				return err
			}
		}
		return nil
	}

	if sel.Group != nil && l.children != nil {
		for _, key := range sel.Group {
			lvl, ok := l.children[key]
			if ok {
				err := lvl.findBuffers(selector[1:], offset, f)
				if err != nil {
					return err
				}
			}
		}
		return nil
	}

	if sel.Any && l.children != nil {
		for _, lvl := range l.children {
			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
				return err
			}
		}
		return nil
	}

	return nil
}

-          b.data = make([]schema.Float, 0, BufferCap)

-    m, empty, err := child.freeAndCheckEmpty(t)


Now let me verify it compiles and run the tests.


Bash

Build metricstore package


(Bash completed with no output)


Compiles cleanly. The `isEmpty()` warning is expected — it's now unused since `freeAndCheckEmpty` handles it inline. Let me check if `isEmpty` is used anywhere else before removing it.


Found 1 file
pkg/metricstore/level.go


Only used in its own definition. Let me remove it.


Edit

// isEmpty returns true if this level has no metrics and no children.
func (l *Level) isEmpty() bool {
	l.lock.RLock()
	defer l.lock.RUnlock()

	for _, b := range l.metrics {
		if b != nil {
			return false
		}
	}
	return len(l.children) == 0
}

// forceFree


// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides level.go: Hierarchical tree structure for metric storage.
//
// # Level Architecture
//
// The Level type forms a tree structure where each node represents a level in the
// ClusterCockpit hierarchy: cluster → host → socket → core → hwthread, with special
// nodes for memory domains and accelerators.
//
// Structure:
//
//	Root Level (cluster="emmy")
//	├─ Level (host="node001")
//	│  ├─ Level (socket="0")
//	│  │  ├─ Level (core="0") [stores cpu0 metrics]
//	│  │  └─ Level (core="1") [stores cpu1 metrics]
//	│  └─ Level (socket="1")
//	│     └─ ...
//	└─ Level (host="node002")
//	   └─ ...
//
// Each Level can:
//   - Hold data (metrics slice of buffer pointers)
//   - Have child nodes (children map[string]*Level)
//   - Both simultaneously (inner nodes can store aggregated metrics)
//
// # Selector Paths
//
// Selectors are hierarchical paths: []string{"cluster", "host", "component"}.
// Example: []string{"emmy", "node001", "cpu0"} navigates to the cpu0 core level.
//
// # Concurrency
//
// RWMutex protects children map and metrics slice. Read-heavy workload (metric reads)
// uses RLock. Writes (new levels, buffer updates) use Lock. Double-checked locking
// prevents races during level creation.
package metricstore

import (
	"sync"
	"time"
	"unsafe"

	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-lib/v2/util"
)

// Level represents a node in the hierarchical metric storage tree.
//
// Can be both a leaf or inner node. Inner nodes hold data in 'metrics' for aggregated
// values (e.g., socket-level metrics derived from core-level data). Named "Level"
// instead of "node" to avoid confusion with cluster nodes (hosts).
//
// Fields:
//   - children: Map of child level names to Level pointers (e.g., "cpu0" → Level)
//   - metrics:  Slice of buffer pointers (one per metric, indexed by MetricConfig.offset)
//   - lock:     RWMutex for concurrent access (read-heavy, write-rare)
type Level struct {
	children map[string]*Level
	metrics  []*buffer
	lock     sync.RWMutex
}

// findLevelOrCreate navigates to or creates the level specified by selector.
//
// Recursively descends the tree, creating missing levels as needed. Uses double-checked
// locking: RLock first (fast path), then Lock if creation needed (slow path), then
// re-check after acquiring Lock to handle races.
//
// Example selector: []string{"emmy", "node001", "cpu0"}
// Navigates: root → emmy → node001 → cpu0, creating levels as needed.
//
// Parameters:
//   - selector: Hierarchical path (consumed recursively, decreasing depth)
//   - nMetrics: Number of metric slots to allocate in new levels
//
// Returns:
//   - *Level: The target level (existing or newly created)
//
// Note: sync.Map may improve performance for high-concurrency writes, but current
// approach suffices for read-heavy workload.
func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
	if len(selector) == 0 {
		return l
	}

	// Allow concurrent reads:
	l.lock.RLock()
	var child *Level
	var ok bool
	if l.children == nil {
		// Children map needs to be created...
		l.lock.RUnlock()
	} else {
		child, ok = l.children[selector[0]]
		l.lock.RUnlock()
		if ok {
			return child.findLevelOrCreate(selector[1:], nMetrics)
		}
	}

	// The level does not exist, take write lock for unique access:
	l.lock.Lock()
	// While this thread waited for the write lock, another thread
	// could have created the child node.
	if l.children != nil {
		child, ok = l.children[selector[0]]
		if ok {
			l.lock.Unlock()
			return child.findLevelOrCreate(selector[1:], nMetrics)
		}
	}

	child = &Level{
		metrics:  make([]*buffer, nMetrics),
		children: nil,
	}

	if l.children != nil {
		l.children[selector[0]] = child
	} else {
		l.children = map[string]*Level{selector[0]: child}
	}
	l.lock.Unlock()
	return child.findLevelOrCreate(selector[1:], nMetrics)
}

// collectPaths gathers all selector paths at the specified depth in the tree.
//
// Recursively traverses children, collecting paths when currentDepth+1 == targetDepth.
// Each path is a selector that can be used with findLevel() or findBuffers().
//
// Explicitly copies slices to avoid shared underlying arrays between siblings, preventing
// unintended mutations.
//
// Parameters:
//   - currentDepth: Depth of current level (0 = root)
//   - targetDepth:  Depth to collect paths from
//   - currentPath:  Path accumulated so far
//   - results:      Output slice (appended to)
//
// Example: collectPaths(0, 2, []string{}, &results) collects all 2-level paths
// like []string{"emmy", "node001"}, []string{"emmy", "node002"}, etc.
func (l *Level) collectPaths(currentDepth, targetDepth int, currentPath []string, results *[][]string) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	for key, child := range l.children {
		if child == nil {
			continue
		}

		// We explicitly make a new slice and copy data to avoid sharing underlying arrays between siblings
		newPath := make([]string, len(currentPath))
		copy(newPath, currentPath)
		newPath = append(newPath, key)

		// Check depth, and just return if depth reached
		if currentDepth+1 == targetDepth {
			*results = append(*results, newPath)
		} else {
			child.collectPaths(currentDepth+1, targetDepth, newPath, results)
		}
	}
}

// free removes buffers older than the retention threshold from the entire subtree.
//
// Recursively frees buffers in this level's metrics and all child levels. Buffers
// with standard capacity (BufferCap) are returned to the pool. Called by the
// retention worker to enforce retention policies.
//
// Parameters:
//   - t: Retention threshold timestamp (Unix seconds)
//
// Returns:
//   - int:   Total number of buffers freed in this subtree
//   - error: Non-nil on failure (propagated from children)
func (l *Level) free(t int64) (int, error) {
	n, _, err := l.freeAndCheckEmpty(t)
	return n, err
}

// freeAndCheckEmpty performs free() and atomically checks if the level is empty
// while still holding its own lock, avoiding a TOCTOU race between free() and
// a separate isEmpty() call.
func (l *Level) freeAndCheckEmpty(t int64) (int, bool, error) {
	l.lock.Lock()
	defer l.lock.Unlock()

	n := 0
	for i, b := range l.metrics {
		if b != nil {
			delme, m := b.free(t)
			n += m
			if delme {
				if cap(b.data) != BufferCap {
					b.data = make([]schema.Float, 0, BufferCap)
				}
				b.lastUsed = time.Now().Unix()
				bufferPool.Put(b)
				l.metrics[i] = nil
			}
		}
	}

	for key, child := range l.children {
		m, empty, err := child.freeAndCheckEmpty(t)
		n += m
		if err != nil {
			return n, false, err
		}
		if empty {
			delete(l.children, key)
		}
	}

	// Check emptiness while still holding the lock
	empty := len(l.children) == 0
	if empty {
		for _, b := range l.metrics {
			if b != nil {
				empty = false
				break
			}
		}
	}

	return n, empty, nil
}

// isEmpty returns true if this level has no metrics and no children.
func (l *Level) isEmpty() bool {
	l.lock.RLock()
	defer l.lock.RUnlock()

	for _, b := range l.metrics {
		if b != nil {
			return false
		}
	}
	return len(l.children) == 0
}

// forceFree removes the oldest buffer from each metric chain in the subtree.
//
// Unlike free(), which removes based on time threshold, this unconditionally removes
// the oldest buffer in each chain. Used by MemoryUsageTracker when memory cap is
// exceeded and time-based retention is insufficient.
//
// Recursively processes current level's metrics and all child levels.
//
// Returns:
//   - int:   Total number of buffers freed in this subtree
//   - error: Non-nil on failure (propagated from children)
func (l *Level) forceFree() (int, error) {
	l.lock.Lock()
	defer l.lock.Unlock()

	n := 0

	// Iterate over metrics in the current level
	for i, b := range l.metrics {
		if b != nil {
			// Attempt to free the oldest buffer in this chain
			delme, freedCount := b.forceFreeOldest()
			n += freedCount

			// If delme is true, it means 'b' itself (the head) was the oldest
			// and needs to be removed from the slice.
			if delme {
				b.next = nil
				b.prev = nil
				if cap(b.data) != BufferCap {
					b.data = make([]schema.Float, 0, BufferCap)
				}
				b.lastUsed = time.Now().Unix()
				bufferPool.Put(b)
				l.metrics[i] = nil
			}
		}
	}

	// Recursively traverse children
	for _, child := range l.children {
		m, err := child.forceFree()
		n += m
		if err != nil {
			return n, err
		}
	}

	return n, nil
}

// sizeInBytes calculates the total memory usage of all buffers in the subtree.
//
// Recursively sums buffer data sizes (count of Float values × sizeof(Float)) across
// this level's metrics and all child levels. Used by MemoryUsageTracker to enforce
// memory cap limits.
//
// Returns:
//   - int64: Total bytes used by buffer data in this subtree
func (l *Level) sizeInBytes() int64 {
	l.lock.RLock()
	defer l.lock.RUnlock()
	size := int64(0)

	for _, b := range l.metrics {
		if b != nil {
			size += b.count() * int64(unsafe.Sizeof(schema.Float(0)))
			size += b.bufferCount() * int64(unsafe.Sizeof(buffer{}))
		}
	}

	for _, child := range l.children {
		size += child.sizeInBytes()
	}

	return size
}

// findLevel navigates to the level specified by selector, returning nil if not found.
//
// Read-only variant of findLevelOrCreate. Does not create missing levels.
// Recursively descends the tree following the selector path.
//
// Parameters:
//   - selector: Hierarchical path (e.g., []string{"emmy", "node001", "cpu0"})
//
// Returns:
//   - *Level: The target level, or nil if any component in the path does not exist
func (l *Level) findLevel(selector []string) *Level {
	if len(selector) == 0 {
		return l
	}

	l.lock.RLock()
	defer l.lock.RUnlock()

	lvl := l.children[selector[0]]
	if lvl == nil {
		return nil
	}

	return lvl.findLevel(selector[1:])
}

// findBuffers invokes callback on all buffers matching the selector pattern.
//
// Supports flexible selector patterns (from cc-lib/util.Selector):
//   - Exact match: Selector element with String set (e.g., "node001")
//   - Group match: Selector element with Group set (e.g., ["cpu0", "cpu2", "cpu4"])
//   - Wildcard:    Selector element with Any=true (matches all children)
//
// Empty selector (len==0) matches current level's buffer at 'offset' and recursively
// all descendant buffers at the same offset (used for aggregation queries).
//
// Parameters:
//   - selector: Pattern to match (consumed recursively)
//   - offset:   Metric index in metrics slice (from MetricConfig.offset)
//   - f:        Callback invoked on each matching buffer
//
// Returns:
//   - error: First error returned by callback, or nil if all succeeded
//
// Example:
//
//	// Find all cpu0 buffers across all hosts:
//	findBuffers([]Selector{{Any: true}, {String: "cpu0"}}, metricOffset, callback)
func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
	l.lock.RLock()
	defer l.lock.RUnlock()

	if len(selector) == 0 {
		b := l.metrics[offset]
		if b != nil {
			return f(b)
		}

		for _, lvl := range l.children {
			err := lvl.findBuffers(nil, offset, f)
			if err != nil {
				return err
			}
		}
		return nil
	}

	sel := selector[0]
	if len(sel.String) != 0 && l.children != nil {
		lvl, ok := l.children[sel.String]
		if ok {
			err := lvl.findBuffers(selector[1:], offset, f)
			if err != nil {
				return err
			}
		}
		return nil
	}

	if sel.Group != nil && l.children != nil {
		for _, key := range sel.Group {
			lvl, ok := l.children[key]
			if ok {
				err := lvl.findBuffers(selector[1:], offset, f)
				if err != nil {
					return err
				}
			}
		}
		return nil
	}

	if sel.Any && l.children != nil {
		for _, lvl := range l.children {
			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
				return err
			}
		}
		return nil
	}

	return nil
}

-// isEmpty returns true if this level has no metrics and no children.


Now run the tests.


Bash

Run metricstore tests


ok  	github.com/ClusterCockpit/cc-backend/pkg/metricstore	0.545s


All tests pass. Let me also check if the config schema validation file needs updating for the new field.


Found 2 files
pkg/metricstore/metricstore.go
pkg/metricstore/configSchema.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→const configSchema = `{
     9→  "type": "object",
    10→  "description": "Configuration specific to built-in metric-store.",
    11→  "properties": {
    12→    "num-workers": {
    13→      "description": "Number of concurrent workers for checkpoint and archive operations",
    14→      "type": "integer"
    15→    },
    16→    "checkpoints": {
    17→      "description": "Configuration for checkpointing the metrics buffers",
    18→      "type": "object",
    19→      "properties": {
    20→        "file-format": {
    21→          "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'wal'.",
    22→          "type": "string"
    23→        },
    24→        "directory": {
    25→          "description": "Path in which the checkpointed files should be placed.",
    26→          "type": "string"
    27→        }
    28→      }
    29→    },
    30→    "cleanup": {
    31→      "description": "Configuration for the cleanup process. The cleanup interval is always 'retention-in-memory'.",
    32→      "type": "object",
    33→      "properties": {
    34→        "mode": {
    35→          "description": "The operation mode (e.g., 'archive' or 'delete').",
    36→          "type": "string",
    37→          "enum": ["archive", "delete"]
    38→        },
    39→        "directory": {
    40→          "description": "Target directory for archive operations.",
    41→          "type": "string"
    42→        }
    43→      },
    44→      "if": {
    45→        "properties": {
    46→          "mode": { "const": "archive" }
    47→        }
    48→      },
    49→      "then": {
    50→        "required": ["directory"]
    51→      }
    52→    },
    53→    "retention-in-memory": {
    54→      "description": "Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.",
    55→      "type": "string"
    56→    },
    57→    "memory-cap": {
    58→      "description": "Upper memory capacity limit used by metricstore in GB",
    59→      "type": "integer"
    60→    },
    61→    "nats-subscriptions": {
    62→      "description": "Array of various subscriptions. Allows to subscribe to different subjects and publishers.",
    63→      "type": "array",
    64→      "items": {
    65→        "type": "object",
    66→        "properties": {
    67→          "subscribe-to": {
    68→            "description": "Subject name",
    69→            "type": "string"
    70→          },
    71→          "cluster-tag": {
    72→            "description": "Optional: Allow lines without a cluster tag, use this as default",
    73→            "type": "string"
    74→          }
    75→        },
    76→				"required": ["subscribe-to"]
    77→      }
    78→    }
    79→  },
    80→  "required": ["retention-in-memory", "memory-cap"]
    81→}`
    82→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

const configSchema = `{
  "type": "object",
  "description": "Configuration specific to built-in metric-store.",
  "properties": {
    "num-workers": {
      "description": "Number of concurrent workers for checkpoint and archive operations",
      "type": "integer"
    },
    "checkpoints": {
      "description": "Configuration for checkpointing the metrics buffers",
      "type": "object",
      "properties": {
        "file-format": {
          "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'wal'.",
          "type": "string"
        },
        "directory": {
          "description": "Path in which the checkpointed files should be placed.",
          "type": "string"
        }
      }
    },
    "cleanup": {
      "description": "Configuration for the cleanup process. The cleanup interval is always 'retention-in-memory'.",
      "type": "object",
      "properties": {
        "mode": {
          "description": "The operation mode (e.g., 'archive' or 'delete').",
          "type": "string",
          "enum": ["archive", "delete"]
        },
        "directory": {
          "description": "Target directory for archive operations.",
          "type": "string"
        }
      },
      "if": {
        "properties": {
          "mode": { "const": "archive" }
        }
      },
      "then": {
        "required": ["directory"]
      }
    },
    "retention-in-memory": {
      "description": "Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.",
      "type": "string"
    },
    "memory-cap": {
      "description": "Upper memory capacity limit used by metricstore in GB",
      "type": "integer"
    },
    "nats-subscriptions": {
      "description": "Array of various subscriptions. Allows to subscribe to different subjects and publishers.",
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "subscribe-to": {
            "description": "Subject name",
            "type": "string"
          },
          "cluster-tag": {
            "description": "Optional: Allow lines without a cluster tag, use this as default",
            "type": "string"
          }
        },
				"required": ["subscribe-to"]
      }
    }
  },
  "required": ["retention-in-memory", "memory-cap"]
}`


Edit

    "memory-cap": {
      "description": "Upper memory capacity limit used by metricstore in GB",
      "type": "integer"
    },

    "checkpoint-interval": {
      "description": "Interval between checkpoints as a Go duration string (e.g., '12h', '6h', '30m'). Default is '12h'.",
      "type": "string"
    },
    "memory-cap": {
      "description": "Upper memory capacity limit used by metricstore in GB",
      "type": "integer"
    },


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/configSchema.go has been updated successfully.

    "checkpoint-interval": {
      "description": "Interval between checkpoints as a Go duration string (e.g., '12h', '6h', '30m'). Default is '12h'.",
      "type": "string"
    },
    "memory-cap": {
      "description": "Upper memory capacity limit used by metricstore in GB",
      "type": "integer"
    },

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

const configSchema = `{
  "type": "object",
  "description": "Configuration specific to built-in metric-store.",
  "properties": {
    "num-workers": {
      "description": "Number of concurrent workers for checkpoint and archive operations",
      "type": "integer"
    },
    "checkpoints": {
      "description": "Configuration for checkpointing the metrics buffers",
      "type": "object",
      "properties": {
        "file-format": {
          "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'wal'.",
          "type": "string"
        },
        "directory": {
          "description": "Path in which the checkpointed files should be placed.",
          "type": "string"
        }
      }
    },
    "cleanup": {
      "description": "Configuration for the cleanup process. The cleanup interval is always 'retention-in-memory'.",
      "type": "object",
      "properties": {
        "mode": {
          "description": "The operation mode (e.g., 'archive' or 'delete').",
          "type": "string",
          "enum": ["archive", "delete"]
        },
        "directory": {
          "description": "Target directory for archive operations.",
          "type": "string"
        }
      },
      "if": {
        "properties": {
          "mode": { "const": "archive" }
        }
      },
      "then": {
        "required": ["directory"]
      }
    },
    "retention-in-memory": {
      "description": "Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.",
      "type": "string"
    },
    "memory-cap": {
      "description": "Upper memory capacity limit used by metricstore in GB",
      "type": "integer"
    },
    "nats-subscriptions": {
      "description": "Array of various subscriptions. Allows to subscribe to different subjects and publishers.",
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "subscribe-to": {
            "description": "Subject name",
            "type": "string"
          },
          "cluster-tag": {
            "description": "Optional: Allow lines without a cluster tag, use this as default",
            "type": "string"
          }
        },
				"required": ["subscribe-to"]
      }
    }
  },
  "required": ["retention-in-memory", "memory-cap"]
}`

+      "description": "Interval between checkpoints as a Go duration string (e.g., '12h', '6h', '30m'). Default is '12h'.",


Bash

go build ./pkg/metricstore/... && go test ./pkg/metricstore/... 2>&1

Build and test metricstore


ok  	github.com/ClusterCockpit/cc-backend/pkg/metricstore	(cached)


All changes compile and tests pass. Here's a summary of what was implemented:

**Bug fixes:**

1. **Bug 1 — TOCTOU race in `free()`** (`level.go`): Replaced separate `free()` + `isEmpty()` calls with a single `freeAndCheckEmpty()` method that checks emptiness while still holding the lock. `free()` now delegates to it. Removed the now-unused `isEmpty()` method.

2. **Bug 2 — Wrong index after buffer switch in `stats()`** (`stats.go:45`): Changed `idx = 0` to `idx = int((t - b.start) / b.frequency)` to correctly recalculate the position in the next buffer, matching the fix already applied in `buffer.read()`.

3. **Bug 3 — `FreeAll()` not thread-safe** (`metricstore.go:756`): Added `m.root.lock.Lock()`/`Unlock()` around the map iteration and deletion.

4. **Bug 6 — `ScaleBy(0)` is a no-op** (`api.go:130`): Changed `f == 0 || f == 1` to `f == 1` so scaling by 0 correctly zeroes out data.

5. **Bug 7 — `collectMetricStatus` wrong metric for unknown names** (`healthcheck.go:69`): Added `mc, ok := m.Metrics[metricName]; if !ok { continue }` to skip unknown metrics (they'll be reported as missing).

**Improvement:**

6. **Configurable checkpoint interval** (`config.go`, `configSchema.go`, `checkpoint.go`): Added `CheckpointInterval string` field to `MetricStoreConfig` (json: `"checkpoint-interval,omitempty"`). When set, `Checkpointing()` parses it with `time.ParseDuration`; defaults to `12h` when empty/unset.


What about the other Improvements. Which ones do you recommend


Let me read the existing plan and the relevant code to evaluate the remaining improvements.


     1→# Metricstore In-Depth Review: Bugs & Improvements
     2→
     3→## BUG 1: `isEmpty()` deadlock risk in `free()` (level.go:209,218)
     4→**Severity: HIGH** — `free()` holds `l.lock.Lock()` (line 184), then calls `child.free(t)` which acquires `child.lock.Lock()`. Then `child.isEmpty()` (line 209) acquires `child.lock.RLock()` — **re-entrant RLock on a locked RWMutex**. Go's `sync.RWMutex` is NOT reentrant. `child.free()` returns with `child.lock` unlocked (defer), then `isEmpty()` re-locks it — this works. BUT: `isEmpty()` is called on the child right after `child.free()` returns, and `child.free()` already released its lock via defer. Between `child.free()` returning and `isEmpty()` being called, another goroutine could add a new metric to the child, making it appear non-empty. This is a **TOCTOU race** but not a deadlock. However, the real risk is: during the `child.free()` call, child's children may have been pruned, and then `isEmpty()` re-acquires the lock — this is safe but has a race window.
     5→
     6→**Fix**: Check emptiness inside `free()` itself, returning a boolean alongside `(int, error)`, so the parent can decide while holding its own lock consistently.
     7→
     8→## BUG 2: `stats()` index recalculation after buffer switch (stats.go:40-46)
     9→**Severity: MEDIUM** — When `idx >= cap(b.data)`, the code does `b = b.next; idx = 0`. This hardcodes idx=0, ignoring the actual timestamp position in the next buffer. If there's a gap between buffers (different start times), `idx=0` is wrong — it reads data from the wrong timestamp. This is the **same bug** that was already fixed in `buffer.read()` (the `TestBufferReadIdxAfterSwitch` test covers read, but NOT stats).
    10→
    11→**Fix**: After `b = b.next`, recalculate `idx = int((t - b.start) / b.frequency)` (same as was done in `read()`).
    12→
    13→## BUG 3: `FreeAll()` not thread-safe (metricstore.go:756-762)
    14→**Severity: MEDIUM** — `FreeAll()` iterates and deletes from `m.root.children` without holding `m.root.lock`. Concurrent reads/writes will race.
    15→
    16→**Fix**: Wrap in `m.root.lock.Lock()`/`Unlock()`.
    17→
    18→## BUG 4: `toCheckpointFile` reads buffer without lock (checkpoint.go:268-299)
    19→**Severity: MEDIUM** — `toCheckpointFile` holds `l.lock.RLock()` (line 257) protecting `l.metrics`, but the buffer's `iterFromTo` and `read` are called without the buffer's own lock. Concurrent `write()` to the same buffer (from NATS ingestion in `WriteToLevel`) can cause data races because `write()` mutates `b.data` (append) and `b.next` under `l.lock.Lock()` — but `toCheckpointFile` only holds `RLock`. This means a write and a checkpoint read can race on buffer internals.
    20→
    21→**Fix**: This is partially mitigated because `read()` and `iterFromTo` only read `b.data` and traverse `b.prev`. The real danger is `b.data` being appended to (which can reallocate the slice) while being read. Since `l.lock.RLock` is held during checkpoint read, and `WriteToLevel` acquires `l.lock.Lock`, they are actually mutually exclusive. **Actually safe** — no fix needed.
    22→
    23→## BUG 5: WAL writes not fsynced (walCheckpoint.go:307)
    24→**Severity: LOW** — `writeWALRecord` does `f.Write()` without `f.Sync()`. On crash, WAL records may be lost even though they were "written". The CRC check on replay handles corrupt trailing records gracefully, but recent data can be silently lost.
    25→
    26→**Fix**: Consider periodic `f.Sync()` (e.g., every N records or on a timer) rather than per-record fsync for performance.
    27→
    28→## BUG 6: `ScaleBy(0)` is a no-op (api.go:131)
    29→**Severity: LOW** — `ScaleBy` returns early when factor is 0, but scaling by 0 should zero out data. The comment says "for performance" but the semantics are wrong.
    30→
    31→**Fix**: Only skip for factor == 1 (identity), not 0.
    32→
    33→## BUG 7: `collectMetricStatus` panics on unknown metric (healthcheck.go:69-70)
    34→**Severity: MEDIUM** — If `expectedMetrics` contains a metric name not present in `m.Metrics`, `mc := m.Metrics[metricName]` returns zero-value MetricConfig with `offset=0`, then `l.metrics[mc.offset]` accesses index 0 which may belong to a different metric. No panic, but **wrong metric data** is checked for health status.
    35→
    36→**Fix**: Check `_, ok := m.Metrics[metricName]` and skip unknown metrics.
    37→
    38→## IMPROVEMENT 1: WAL buffering (walCheckpoint.go:167-176)
    39→**Current**: Each WAL message results in a separate `f.Write()` syscall. With thousands of metrics per second, this is excessive.
    40→**Fix**: Use a `bufio.Writer` per host WAL file, flushing periodically or on rotation.
    41→
    42→## IMPROVEMENT 2: Checkpoint interval hardcoded (checkpoint.go:103)
    43→**Current**: `const checkpointInterval = 12 * time.Hour` — not configurable.
    44→**Fix**: Add optional `CheckpointInterval string` to `MetricStoreConfig` (json: `"checkpoint-interval",omitempty`). Default to `12h` when empty/unset. Parse with `time.ParseDuration` in `Checkpointing()`.
    45→
    46→## IMPROVEMENT 3: `MemoryCap` type mismatch
    47→**Current**: `MemoryCap int` in config (bytes implied) but compared as GB: `float64(Keys.MemoryCap)` vs `actualMemoryGB`. If user sets `"memory-cap": 32`, it means 32 GB, not 32 bytes. The config comment says "bytes" but the code treats it as GB.
    48→**Fix**: Clarify the unit in docs/config schema; it's actually GB.
    49→
    50→## IMPROVEMENT 4: No WAL size limit
    51→**Current**: WAL files grow unbounded between checkpoints (12h). If ingestion rate is high, WAL files can consume significant disk space and slow replay.
    52→**Fix**: Add configurable WAL size limit; trigger checkpoint when exceeded.
    53→
    54→---
    55→
    56→## Files to modify
    57→
    58→- `pkg/metricstore/level.go` — Bug 1: `free()` returns `(int, bool, error)` where bool=isEmpty; remove separate `isEmpty()` call
    59→- `pkg/metricstore/stats.go:46` — Bug 2: recalculate `idx = int((t - b.start) / b.frequency)` after `b = b.next`
    60→- `pkg/metricstore/metricstore.go:756` — Bug 3: wrap `FreeAll()` body in `m.root.lock.Lock()`/`Unlock()`
    61→- `pkg/metricstore/api.go:131` — Bug 6: change `f == 0 || f == 1` to `f == 1`
    62→- `pkg/metricstore/healthcheck.go:69` — Bug 7: add `mc, ok := m.Metrics[metricName]; if !ok { continue }`
    63→- `pkg/metricstore/config.go` — Improvement 2: add optional `CheckpointInterval string` field (json `"checkpoint-interval,omitempty"`)
    64→- `pkg/metricstore/checkpoint.go:103` — Improvement 2: use `Keys.CheckpointInterval` if set, else default `12h`
    65→
    66→## Verification
    67→```bash
    68→go test ./pkg/metricstore/...
    69→```
    70→

# Metricstore In-Depth Review: Bugs & Improvements

## BUG 1: `isEmpty()` deadlock risk in `free()` (level.go:209,218)
**Severity: HIGH** — `free()` holds `l.lock.Lock()` (line 184), then calls `child.free(t)` which acquires `child.lock.Lock()`. Then `child.isEmpty()` (line 209) acquires `child.lock.RLock()` — **re-entrant RLock on a locked RWMutex**. Go's `sync.RWMutex` is NOT reentrant. `child.free()` returns with `child.lock` unlocked (defer), then `isEmpty()` re-locks it — this works. BUT: `isEmpty()` is called on the child right after `child.free()` returns, and `child.free()` already released its lock via defer. Between `child.free()` returning and `isEmpty()` being called, another goroutine could add a new metric to the child, making it appear non-empty. This is a **TOCTOU race** but not a deadlock. However, the real risk is: during the `child.free()` call, child's children may have been pruned, and then `isEmpty()` re-acquires the lock — this is safe but has a race window.

**Fix**: Check emptiness inside `free()` itself, returning a boolean alongside `(int, error)`, so the parent can decide while holding its own lock consistently.

## BUG 2: `stats()` index recalculation after buffer switch (stats.go:40-46)
**Severity: MEDIUM** — When `idx >= cap(b.data)`, the code does `b = b.next; idx = 0`. This hardcodes idx=0, ignoring the actual timestamp position in the next buffer. If there's a gap between buffers (different start times), `idx=0` is wrong — it reads data from the wrong timestamp. This is the **same bug** that was already fixed in `buffer.read()` (the `TestBufferReadIdxAfterSwitch` test covers read, but NOT stats).

**Fix**: After `b = b.next`, recalculate `idx = int((t - b.start) / b.frequency)` (same as was done in `read()`).

## BUG 3: `FreeAll()` not thread-safe (metricstore.go:756-762)
**Severity: MEDIUM** — `FreeAll()` iterates and deletes from `m.root.children` without holding `m.root.lock`. Concurrent reads/writes will race.

**Fix**: Wrap in `m.root.lock.Lock()`/`Unlock()`.

## BUG 4: `toCheckpointFile` reads buffer without lock (checkpoint.go:268-299)
**Severity: MEDIUM** — `toCheckpointFile` holds `l.lock.RLock()` (line 257) protecting `l.metrics`, but the buffer's `iterFromTo` and `read` are called without the buffer's own lock. Concurrent `write()` to the same buffer (from NATS ingestion in `WriteToLevel`) can cause data races because `write()` mutates `b.data` (append) and `b.next` under `l.lock.Lock()` — but `toCheckpointFile` only holds `RLock`. This means a write and a checkpoint read can race on buffer internals.

**Fix**: This is partially mitigated because `read()` and `iterFromTo` only read `b.data` and traverse `b.prev`. The real danger is `b.data` being appended to (which can reallocate the slice) while being read. Since `l.lock.RLock` is held during checkpoint read, and `WriteToLevel` acquires `l.lock.Lock`, they are actually mutually exclusive. **Actually safe** — no fix needed.

## BUG 5: WAL writes not fsynced (walCheckpoint.go:307)
**Severity: LOW** — `writeWALRecord` does `f.Write()` without `f.Sync()`. On crash, WAL records may be lost even though they were "written". The CRC check on replay handles corrupt trailing records gracefully, but recent data can be silently lost.

**Fix**: Consider periodic `f.Sync()` (e.g., every N records or on a timer) rather than per-record fsync for performance.

## BUG 6: `ScaleBy(0)` is a no-op (api.go:131)
**Severity: LOW** — `ScaleBy` returns early when factor is 0, but scaling by 0 should zero out data. The comment says "for performance" but the semantics are wrong.

**Fix**: Only skip for factor == 1 (identity), not 0.

## BUG 7: `collectMetricStatus` panics on unknown metric (healthcheck.go:69-70)
**Severity: MEDIUM** — If `expectedMetrics` contains a metric name not present in `m.Metrics`, `mc := m.Metrics[metricName]` returns zero-value MetricConfig with `offset=0`, then `l.metrics[mc.offset]` accesses index 0 which may belong to a different metric. No panic, but **wrong metric data** is checked for health status.

**Fix**: Check `_, ok := m.Metrics[metricName]` and skip unknown metrics.

## IMPROVEMENT 1: WAL buffering (walCheckpoint.go:167-176)
**Current**: Each WAL message results in a separate `f.Write()` syscall. With thousands of metrics per second, this is excessive.
**Fix**: Use a `bufio.Writer` per host WAL file, flushing periodically or on rotation.

## IMPROVEMENT 2: Checkpoint interval hardcoded (checkpoint.go:103)
**Current**: `const checkpointInterval = 12 * time.Hour` — not configurable.
**Fix**: Add optional `CheckpointInterval string` to `MetricStoreConfig` (json: `"checkpoint-interval",omitempty`). Default to `12h` when empty/unset. Parse with `time.ParseDuration` in `Checkpointing()`.

## IMPROVEMENT 3: `MemoryCap` type mismatch
**Current**: `MemoryCap int` in config (bytes implied) but compared as GB: `float64(Keys.MemoryCap)` vs `actualMemoryGB`. If user sets `"memory-cap": 32`, it means 32 GB, not 32 bytes. The config comment says "bytes" but the code treats it as GB.
**Fix**: Clarify the unit in docs/config schema; it's actually GB.

## IMPROVEMENT 4: No WAL size limit
**Current**: WAL files grow unbounded between checkpoints (12h). If ingestion rate is high, WAL files can consume significant disk space and slow replay.
**Fix**: Add configurable WAL size limit; trigger checkpoint when exceeded.

---

## Files to modify

- `pkg/metricstore/level.go` — Bug 1: `free()` returns `(int, bool, error)` where bool=isEmpty; remove separate `isEmpty()` call
- `pkg/metricstore/stats.go:46` — Bug 2: recalculate `idx = int((t - b.start) / b.frequency)` after `b = b.next`
- `pkg/metricstore/metricstore.go:756` — Bug 3: wrap `FreeAll()` body in `m.root.lock.Lock()`/`Unlock()`
- `pkg/metricstore/api.go:131` — Bug 6: change `f == 0 || f == 1` to `f == 1`
- `pkg/metricstore/healthcheck.go:69` — Bug 7: add `mc, ok := m.Metrics[metricName]; if !ok { continue }`
- `pkg/metricstore/config.go` — Improvement 2: add optional `CheckpointInterval string` field (json `"checkpoint-interval,omitempty"`)
- `pkg/metricstore/checkpoint.go:103` — Improvement 2: use `Keys.CheckpointInterval` if set, else default `12h`

## Verification
```bash
go test ./pkg/metricstore/...
```


Let me look at the WAL code to properly assess Improvements 1, 4, and 5 (Bug 5 — periodic fsync).


   140→				cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   141→				return nil
   142→			}
   143→
   144→			walPath := path.Join(hostDir, "current.wal")
   145→			f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   146→			if err != nil {
   147→				cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   148→				return nil
   149→			}
   150→
   151→			// Write file header magic if file is new (empty).
   152→			info, err := f.Stat()
   153→			if err == nil && info.Size() == 0 {
   154→				var hdr [4]byte
   155→				binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   156→				if _, err := f.Write(hdr[:]); err != nil {
   157→					cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   158→					f.Close()
   159→					return nil
   160→				}
   161→			}
   162→
   163→			hostFiles[hostDir] = &walFileState{f: f}
   164→			return f
   165→		}
   166→
   167→		processMsg := func(msg *WALMessage) {
   168→			hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   169→			f := getOrOpenWAL(hostDir)
   170→			if f == nil {
   171→				return
   172→			}
   173→			if err := writeWALRecord(f, msg); err != nil {
   174→				cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   175→			}
   176→		}
   177→
   178→		processRotate := func(req walRotateReq) {
   179→			ws, ok := hostFiles[req.hostDir]
   180→			if ok && ws.f != nil {
   181→				ws.f.Close()
   182→				walPath := path.Join(req.hostDir, "current.wal")
   183→				if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   184→					cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   185→				}
   186→				delete(hostFiles, req.hostDir)
   187→			}
   188→			close(req.done)
   189→		}
   190→
   191→		drain := func() {
   192→			for {
   193→				select {
   194→				case msg, ok := <-WALMessages:
   195→					if !ok {
   196→						return
   197→					}
   198→					processMsg(msg)
   199→				case req := <-walRotateCh:
   200→					processRotate(req)
   201→				default:
   202→					return
   203→				}
   204→			}
   205→		}
   206→
   207→		for {
   208→			select {
   209→			case <-ctx.Done():
   210→				drain()
   211→				return
   212→			case msg, ok := <-WALMessages:
   213→				if !ok {
   214→					return
   215→				}
   216→				processMsg(msg)
   217→			case req := <-walRotateCh:
   218→				processRotate(req)
   219→			}
   220→		}
   221→	})
   222→}
   223→
   224→// RotateWALFiles sends rotation requests for the given host directories
   225→// and blocks until all rotations complete.
   226→func RotateWALFiles(hostDirs []string) {
   227→	dones := make([]chan struct{}, len(hostDirs))
   228→	for i, dir := range hostDirs {
   229→		dones[i] = make(chan struct{})

				cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
				return nil
			}

			walPath := path.Join(hostDir, "current.wal")
			f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
			if err != nil {
				cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
				return nil
			}

			// Write file header magic if file is new (empty).
			info, err := f.Stat()
			if err == nil && info.Size() == 0 {
				var hdr [4]byte
				binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
				if _, err := f.Write(hdr[:]); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
					f.Close()
					return nil
				}
			}

			hostFiles[hostDir] = &walFileState{f: f}
			return f
		}

		processMsg := func(msg *WALMessage) {
			hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
			f := getOrOpenWAL(hostDir)
			if f == nil {
				return
			}
			if err := writeWALRecord(f, msg); err != nil {
				cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
			}
		}

		processRotate := func(req walRotateReq) {
			ws, ok := hostFiles[req.hostDir]
			if ok && ws.f != nil {
				ws.f.Close()
				walPath := path.Join(req.hostDir, "current.wal")
				if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
					cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
				}
				delete(hostFiles, req.hostDir)
			}
			close(req.done)
		}

		drain := func() {
			for {
				select {
				case msg, ok := <-WALMessages:
					if !ok {
						return
					}
					processMsg(msg)
				case req := <-walRotateCh:
					processRotate(req)
				default:
					return
				}
			}
		}

		for {
			select {
			case <-ctx.Done():
				drain()
				return
			case msg, ok := <-WALMessages:
				if !ok {
					return
				}
				processMsg(msg)
			case req := <-walRotateCh:
				processRotate(req)
			}
		}
	})
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete.
func RotateWALFiles(hostDirs []string) {
	dones := make([]chan struct{}, len(hostDirs))
	for i, dir := range hostDirs {
		dones[i] = make(chan struct{})


   280→	buf = append(buf, val[:]...)
   281→
   282→	return buf
   283→}
   284→
   285→// writeWALRecord appends a binary WAL record to the file.
   286→// Format: [4B magic][4B payload_len][payload][4B CRC32]
   287→func writeWALRecord(f *os.File, msg *WALMessage) error {
   288→	payload := buildWALPayload(msg)
   289→	crc := crc32.ChecksumIEEE(payload)
   290→
   291→	record := make([]byte, 0, 4+4+len(payload)+4)
   292→
   293→	var magic [4]byte
   294→	binary.LittleEndian.PutUint32(magic[:], walRecordMagic)
   295→	record = append(record, magic[:]...)
   296→
   297→	var pLen [4]byte
   298→	binary.LittleEndian.PutUint32(pLen[:], uint32(len(payload)))
   299→	record = append(record, pLen[:]...)
   300→
   301→	record = append(record, payload...)
   302→
   303→	var crcBytes [4]byte
   304→	binary.LittleEndian.PutUint32(crcBytes[:], crc)
   305→	record = append(record, crcBytes[:]...)
   306→
   307→	_, err := f.Write(record)
   308→	return err
   309→}
   310→
   311→// readWALRecord reads one WAL record from the reader.
   312→// Returns (nil, nil) on clean EOF. Returns error on data corruption.
   313→// A CRC mismatch indicates a truncated trailing record (expected on crash).
   314→func readWALRecord(r io.Reader) (*WALMessage, error) {
   315→	var magic uint32
   316→	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
   317→		if err == io.EOF {
   318→			return nil, nil // Clean EOF
   319→		}
   320→		return nil, fmt.Errorf("read record magic: %w", err)
   321→	}
   322→
   323→	if magic != walRecordMagic {
   324→		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
   325→	}
   326→
   327→	var payloadLen uint32
   328→	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
   329→		return nil, fmt.Errorf("read payload length: %w", err)
   330→	}
   331→
   332→	if payloadLen > 1<<20 { // 1 MB sanity limit
   333→		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
   334→	}
   335→
   336→	payload := make([]byte, payloadLen)
   337→	if _, err := io.ReadFull(r, payload); err != nil {
   338→		return nil, fmt.Errorf("read payload: %w", err)
   339→	}

	buf = append(buf, val[:]...)

	return buf
}

// writeWALRecord appends a binary WAL record to the file.
// Format: [4B magic][4B payload_len][payload][4B CRC32]
func writeWALRecord(f *os.File, msg *WALMessage) error {
	payload := buildWALPayload(msg)
	crc := crc32.ChecksumIEEE(payload)

	record := make([]byte, 0, 4+4+len(payload)+4)

	var magic [4]byte
	binary.LittleEndian.PutUint32(magic[:], walRecordMagic)
	record = append(record, magic[:]...)

	var pLen [4]byte
	binary.LittleEndian.PutUint32(pLen[:], uint32(len(payload)))
	record = append(record, pLen[:]...)

	record = append(record, payload...)

	var crcBytes [4]byte
	binary.LittleEndian.PutUint32(crcBytes[:], crc)
	record = append(record, crcBytes[:]...)

	_, err := f.Write(record)
	return err
}

// readWALRecord reads one WAL record from the reader.
// Returns (nil, nil) on clean EOF. Returns error on data corruption.
// A CRC mismatch indicates a truncated trailing record (expected on crash).
func readWALRecord(r io.Reader) (*WALMessage, error) {
	var magic uint32
	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
		if err == io.EOF {
			return nil, nil // Clean EOF
		}
		return nil, fmt.Errorf("read record magic: %w", err)
	}

	if magic != walRecordMagic {
		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
	}

	var payloadLen uint32
	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
		return nil, fmt.Errorf("read payload length: %w", err)
	}

	if payloadLen > 1<<20 { // 1 MB sanity limit
		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
	}

	payload := make([]byte, payloadLen)
	if _, err := io.ReadFull(r, payload); err != nil {
		return nil, fmt.Errorf("read payload: %w", err)
	}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
     7→//
     8→// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
     9→// per host, eliminating the extra memory overhead of the AvroStore and providing
    10→// truly continuous (per-write) crash safety.
    11→//
    12→// # Architecture
    13→//
    14→//	Metric write (DecodeLine)
    15→//	      │
    16→//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
    17→//	      │
    18→//	      └─► WALMessages channel
    19→//	               │
    20→//	               ▼
    21→//	        WALStaging goroutine
    22→//	               │
    23→//	               ▼
    24→//	        checkpoints/cluster/host/current.wal  (append-only, binary)
    25→//
    26→//	Periodic checkpoint (Checkpointing goroutine):
    27→//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
    28→//	  2. Signal WALStaging to truncate current.wal per host
    29→//
    30→//	On restart (FromCheckpoint):
    31→//	  1. Load most recent <timestamp>.bin snapshot
    32→//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
    33→//
    34→// # WAL Record Format
    35→//
    36→//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
    37→//
    38→//	payload:
    39→//	  [8B timestamp int64]
    40→//	  [2B metric_name_len uint16][N metric name bytes]
    41→//	  [1B selector_count uint8]
    42→//	  per selector: [1B selector_len uint8][M selector bytes]
    43→//	  [4B value float32 bits]
    44→//
    45→// # Binary Snapshot Format
    46→//
    47→//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
    48→//	Level tree (recursive):
    49→//	  [4B num_metrics uint32]
    50→//	  per metric:

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼
//	        WALStaging goroutine
//	               │
//	               ▼
//	        checkpoints/cluster/host/current.wal  (append-only, binary)
//
//	Periodic checkpoint (Checkpointing goroutine):
//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
//	  2. Signal WALStaging to truncate current.wal per host
//
//	On restart (FromCheckpoint):
//	  1. Load most recent <timestamp>.bin snapshot
//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
//
// # WAL Record Format
//
//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
//
//	payload:
//	  [8B timestamp int64]
//	  [2B metric_name_len uint16][N metric name bytes]
//	  [1B selector_count uint8]
//	  per selector: [1B selector_len uint8][M selector bytes]
//	  [4B value float32 bits]
//
// # Binary Snapshot Format
//
//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
//	Level tree (recursive):
//	  [4B num_metrics uint32]
//	  per metric:


    50→//	  per metric:
    51→//	    [2B name_len uint16][N name bytes]
    52→//	    [8B frequency int64][8B start int64]
    53→//	    [4B num_values uint32][num_values × 4B float32]
    54→//	  [4B num_children uint32]
    55→//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
    56→package metricstore
    57→
    58→import (
    59→	"bufio"
    60→	"context"
    61→	"encoding/binary"
    62→	"fmt"
    63→	"hash/crc32"
    64→	"io"
    65→	"math"
    66→	"os"
    67→	"path"
    68→	"strings"
    69→	"sync"
    70→	"sync/atomic"
    71→
    72→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    73→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    74→)
    75→
    76→// Magic numbers for binary formats.
    77→const (
    78→	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
    79→	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
    80→	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
    81→)
    82→
    83→// WALMessages is the channel for sending metric writes to the WAL staging goroutine.
    84→// Buffered to allow burst writes without blocking the metric ingestion path.
    85→var WALMessages = make(chan *WALMessage, 4096)
    86→
    87→// walRotateCh is used by the checkpoint goroutine to request WAL file rotation
    88→// (close, delete, reopen) after a binary snapshot has been written.
    89→var walRotateCh = make(chan walRotateReq, 256)
    90→
    91→// WALMessage represents a single metric write to be appended to the WAL.
    92→// Cluster and Node are NOT stored in the WAL record (inferred from file path).
    93→type WALMessage struct {
    94→	MetricName string
    95→	Cluster    string
    96→	Node       string
    97→	Selector   []string
    98→	Value      schema.Float
    99→	Timestamp  int64
   100→}
   101→
   102→// walRotateReq requests WAL file rotation for a specific host directory.
   103→// The done channel is closed by the WAL goroutine when rotation is complete.
   104→type walRotateReq struct {
   105→	hostDir string
   106→	done    chan struct{}
   107→}
   108→
   109→// walFileState holds an open WAL file handle for one host directory.
   110→type walFileState struct {
   111→	f *os.File
   112→}
   113→
   114→// WALStaging starts a background goroutine that receives WALMessage items
   115→// and appends binary WAL records to per-host current.wal files.
   116→// Also handles WAL rotation requests from the checkpoint goroutine.
   117→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   118→	wg.Go(func() {
   119→		if Keys.Checkpoints.FileFormat == "json" {
   120→			return
   121→		}
   122→
   123→		hostFiles := make(map[string]*walFileState)
   124→
   125→		defer func() {
   126→			for _, ws := range hostFiles {
   127→				if ws.f != nil {
   128→					ws.f.Close()
   129→				}
   130→			}
   131→		}()
   132→
   133→		getOrOpenWAL := func(hostDir string) *os.File {
   134→			ws, ok := hostFiles[hostDir]
   135→			if ok {
   136→				return ws.f
   137→			}
   138→
   139→			if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {

//	  per metric:
//	    [2B name_len uint16][N name bytes]
//	    [8B frequency int64][8B start int64]
//	    [4B num_values uint32][num_values × 4B float32]
//	  [4B num_children uint32]
//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
package metricstore

import (
	"bufio"
	"context"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"io"
	"math"
	"os"
	"path"
	"strings"
	"sync"
	"sync/atomic"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Magic numbers for binary formats.
const (
	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
)

// WALMessages is the channel for sending metric writes to the WAL staging goroutine.
// Buffered to allow burst writes without blocking the metric ingestion path.
var WALMessages = make(chan *WALMessage, 4096)

// walRotateCh is used by the checkpoint goroutine to request WAL file rotation
// (close, delete, reopen) after a binary snapshot has been written.
var walRotateCh = make(chan walRotateReq, 256)

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle for one host directory.
type walFileState struct {
	f *os.File
}

// WALStaging starts a background goroutine that receives WALMessage items
// and appends binary WAL records to per-host current.wal files.
// Also handles WAL rotation requests from the checkpoint goroutine.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	wg.Go(func() {
		if Keys.Checkpoints.FileFormat == "json" {
			return
		}

		hostFiles := make(map[string]*walFileState)

		defer func() {
			for _, ws := range hostFiles {
				if ws.f != nil {
					ws.f.Close()
				}
			}
		}()

		getOrOpenWAL := func(hostDir string) *os.File {
			ws, ok := hostFiles[hostDir]
			if ok {
				return ws.f
			}

			if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {


Now I have a clear picture. Here's my assessment of the remaining improvements:

**Improvement 1 (WAL buffering)** — **Recommend: YES.** Each metric write does a separate `f.Write()` syscall (line 307). With thousands of metrics/sec across hundreds of hosts, this is a significant I/O bottleneck. The `walFileState` struct already exists and can hold a `bufio.Writer`. The WAL staging goroutine is single-threaded, so no extra synchronization needed. Flush on rotation and on shutdown. Low risk, high impact.

**Improvement 3 (MemoryCap docs)** — **Recommend: YES, trivial.** The config schema already says "in GB" (line 58 of configSchema.go), but the struct comment in config.go says "bytes". Just fix the comment. One line change.

**Improvement 4 (WAL size limit)** — **Recommend: NO, not yet.** This adds significant complexity (tracking file sizes per host, triggering mid-cycle checkpoints, coordinating with the checkpoint goroutine). The checkpoint interval is now configurable (Improvement 2, already done), so users can reduce WAL accumulation by shortening the interval. The ROI isn't there until there's evidence of actual WAL size problems in production.

**Bug 5 (periodic fsync)** — **Recommend: NO for now.** The CRC check already handles corrupt trailing records gracefully on crash. Adding periodic fsync adds complexity and I/O overhead. The WAL is a best-effort crash recovery mechanism on top of the binary snapshots — losing a few seconds of data on crash is acceptable. If durability requirements increase, revisit.

So I'd recommend implementing **Improvement 1** (WAL buffering) and **Improvement 3** (fix the misleading comment). Let me write the plan.


   108→// MetricStoreConfig defines the main configuration for the metricstore.
   109→//
   110→// Loaded from cc-backend's config.json "metricstore" section. Controls memory usage,
   111→// persistence, archiving, and metric ingestion.
   112→//
   113→// Fields:
   114→//   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
   115→//   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
   116→//   - MemoryCap:         Max bytes for buffer data (0 = unlimited); triggers forceFree when exceeded
   117→//   - Checkpoints:       Periodic persistence configuration
   118→//   - Debug:             Development/profiling options (nil = disabled)
   119→//   - Archive:           Long-term storage configuration (nil = disabled)
   120→//   - Subscriptions:     NATS topics for metric ingestion (nil = polling only)
   121→type MetricStoreConfig struct {
   122→	// Number of concurrent workers for checkpoint and archive operations.
   123→	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
   124→	NumWorkers         int            `json:"num-workers"`
   125→	RetentionInMemory  string         `json:"retention-in-memory"`
   126→	CheckpointInterval string         `json:"checkpoint-interval,omitempty"`
   127→	MemoryCap          int            `json:"memory-cap"`
   128→	Checkpoints        Checkpoints    `json:"checkpoints"`
   129→	Debug              *Debug         `json:"debug"`
   130→	Cleanup            *Cleanup       `json:"cleanup"`
   131→	Subscriptions      *Subscriptions `json:"nats-subscriptions"`
   132→}

// MetricStoreConfig defines the main configuration for the metricstore.
//
// Loaded from cc-backend's config.json "metricstore" section. Controls memory usage,
// persistence, archiving, and metric ingestion.
//
// Fields:
//   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
//   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
//   - MemoryCap:         Max bytes for buffer data (0 = unlimited); triggers forceFree when exceeded
//   - Checkpoints:       Periodic persistence configuration
//   - Debug:             Development/profiling options (nil = disabled)
//   - Archive:           Long-term storage configuration (nil = disabled)
//   - Subscriptions:     NATS topics for metric ingestion (nil = polling only)
type MetricStoreConfig struct {
	// Number of concurrent workers for checkpoint and archive operations.
	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
	NumWorkers         int            `json:"num-workers"`
	RetentionInMemory  string         `json:"retention-in-memory"`
	CheckpointInterval string         `json:"checkpoint-interval,omitempty"`
	MemoryCap          int            `json:"memory-cap"`
	Checkpoints        Checkpoints    `json:"checkpoints"`
	Debug              *Debug         `json:"debug"`
	Cleanup            *Cleanup       `json:"cleanup"`
	Subscriptions      *Subscriptions `json:"nats-subscriptions"`
}