     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements checkpoint persistence for the in-memory metric store.
     7→//
     8→// Checkpoints enable graceful restarts by periodically saving in-memory metric
     9→// data to disk in JSON or binary format. The checkpoint system:
    10→//
    11→// Key Features:
    12→//   - Periodic background checkpointing via the Checkpointing() worker
    13→//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
    14→//   - Parallel checkpoint creation and loading using worker pools
    15→//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
    16→//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
    17→//   - Only saves unarchived data (archived data is already persisted elsewhere)
    18→//   - GC optimization during loading to prevent excessive heap growth
    19→//
    20→// Checkpoint Workflow:
    21→//  1. Init() loads checkpoints within retention window at startup
    22→//  2. Checkpointing() worker periodically saves new data
    23→//  3. Shutdown() writes final checkpoint before exit
    24→//
    25→// File Organization:
    26→//
    27→//	checkpoints/
    28→//	  cluster1/
    29→//	    host001/
    30→//	      1234567890.json  (JSON format: full subtree snapshot)
    31→//	      1234567890.bin   (binary format: full subtree snapshot)
    32→//	      current.wal      (WAL format: append-only per-entry log)
    33→//	    host002/
    34→//	      ...
    35→package metricstore
    36→
    37→import (
    38→	"bufio"
    39→	"context"
    40→	"encoding/json"
    41→	"errors"
    42→	"fmt"
    43→	"io/fs"
    44→	"os"
    45→	"path"
    46→	"path/filepath"
    47→	"sort"
    48→	"strconv"
    49→	"strings"
    50→	"sync"
    51→	"sync/atomic"
    52→	"time"
    53→
    54→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    55→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    56→)
    57→
    58→const (
    59→	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
    60→	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
    61→	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
    62→)
    63→
    64→// CheckpointMetrics represents metric data in a checkpoint file.
    65→// Whenever the structure changes, update MarshalJSON as well!
    66→type CheckpointMetrics struct {
    67→	Data      []schema.Float `json:"data"`
    68→	Frequency int64          `json:"frequency"`
    69→	Start     int64          `json:"start"`
    70→}
    71→
    72→// CheckpointFile represents the hierarchical structure of a checkpoint file.
    73→// It mirrors the Level tree structure from the MemoryStore.
    74→type CheckpointFile struct {
    75→	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
    76→	Children map[string]*CheckpointFile    `json:"children"`
    77→	From     int64                         `json:"from"`
    78→	To       int64                         `json:"to"`
    79→}
    80→
    81→// lastCheckpoint tracks the timestamp of the last checkpoint creation.
    82→var (
    83→	lastCheckpoint   time.Time
    84→	lastCheckpointMu sync.Mutex
    85→)
    86→
    87→// Checkpointing starts a background worker that periodically saves metric data to disk.
    88→//
    89→// Checkpoints are written every 12 hours (hardcoded).
    90→//
    91→// Format behaviour:
    92→//   - "json": Periodic checkpointing every checkpointInterval
    93→//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
    94→func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
    95→	lastCheckpointMu.Lock()
    96→	lastCheckpoint = time.Now()
    97→	lastCheckpointMu.Unlock()
    98→
    99→	ms := GetMemoryStore()
   100→
   101→	wg.Go(func() {
   102→
   103→		d := 12 * time.Hour // default checkpoint interval
   104→		if Keys.CheckpointInterval != "" {
   105→			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
   106→			if err != nil {
   107→				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
   108→			} else {
   109→				d = parsed
   110→			}
   111→		}
   112→
   113→		ticker := time.NewTicker(d)
   114→		defer ticker.Stop()
   115→
   116→		for {
   117→			select {
   118→			case <-ctx.Done():
   119→				return
   120→			case <-ticker.C:
   121→				lastCheckpointMu.Lock()
   122→				from := lastCheckpoint
   123→				lastCheckpointMu.Unlock()
   124→
   125→				now := time.Now()
   126→				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
   127→
   128→				if Keys.Checkpoints.FileFormat == "wal" {
   129→					// Pause WAL writes: the binary snapshot captures all in-memory
   130→					// data, so WAL records written during checkpoint are redundant
   131→					// and would be deleted during rotation anyway.
   132→					walCheckpointActive.Store(true)
   133→					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   134→					if err != nil {
   135→						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
   136→					}
   137→					if n > 0 {
   138→						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
   139→						lastCheckpointMu.Lock()
   140→						lastCheckpoint = now
   141→						lastCheckpointMu.Unlock()
   142→						// Rotate WAL files for successfully checkpointed hosts.
   143→						RotateWALFiles(hostDirs)
   144→					}
   145→					walCheckpointActive.Store(false)
   146→					walDropped.Store(0)
   147→				} else {
   148→					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   149→					if err != nil {
   150→						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
   151→					} else {
   152→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
   153→						lastCheckpointMu.Lock()
   154→						lastCheckpoint = now
   155→						lastCheckpointMu.Unlock()
   156→					}
   157→				}
   158→			}
   159→		}
   160→	})
   161→}
   162→
   163→// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
   164→//
   165→// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
   166→// This method manually constructs JSON to avoid allocations and interface conversions.
   167→func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
   168→	buf := make([]byte, 0, 128+len(cm.Data)*8)
   169→	buf = append(buf, `{"frequency":`...)
   170→	buf = strconv.AppendInt(buf, cm.Frequency, 10)
   171→	buf = append(buf, `,"start":`...)
   172→	buf = strconv.AppendInt(buf, cm.Start, 10)
   173→	buf = append(buf, `,"data":[`...)
   174→	for i, x := range cm.Data {
   175→		if i != 0 {
   176→			buf = append(buf, ',')
   177→		}
   178→		if x.IsNaN() {
   179→			buf = append(buf, `null`...)
   180→		} else {
   181→			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
   182→		}
   183→	}
   184→	buf = append(buf, `]}`...)
   185→	return buf, nil
   186→}
   187→
   188→// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format).
   189→//
   190→// Metrics at root and cluster levels are skipped. One file per host is created.
   191→// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
   192→// at a time, allowing concurrent writes/reads to other hosts.
   193→//
   194→// Returns the number of checkpoint files created and any errors encountered.
   195→func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
   196→	// Pre-calculate capacity by counting cluster/host pairs
   197→	m.root.lock.RLock()
   198→	totalHosts := 0
   199→	for _, l1 := range m.root.children {
   200→		l1.lock.RLock()
   201→		totalHosts += len(l1.children)
   202→		l1.lock.RUnlock()
   203→	}
   204→	m.root.lock.RUnlock()
   205→
   206→	levels := make([]*Level, 0, totalHosts)
   207→	selectors := make([][]string, 0, totalHosts)
   208→
   209→	m.root.lock.RLock()
   210→	for sel1, l1 := range m.root.children {
   211→		l1.lock.RLock()
   212→		for sel2, l2 := range l1.children {
   213→			levels = append(levels, l2)
   214→			selectors = append(selectors, []string{sel1, sel2})
   215→		}
   216→		l1.lock.RUnlock()
   217→	}
   218→	m.root.lock.RUnlock()
   219→
   220→	type workItem struct {
   221→		level    *Level
   222→		dir      string
   223→		selector []string
   224→	}
   225→
   226→	n, errs := int32(0), int32(0)
   227→
   228→	var wg sync.WaitGroup
   229→	wg.Add(Keys.NumWorkers)
   230→	work := make(chan workItem, Keys.NumWorkers*2)
   231→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   232→		go func() {
   233→			defer wg.Done()
   234→
   235→			for workItem := range work {
   236→				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
   237→					if err == ErrNoNewArchiveData {
   238→						continue
   239→					}
   240→
   241→					cclog.Errorf("[METRICSTORE]> error while checkpointing %#v: %s", workItem.selector, err.Error())
   242→					atomic.AddInt32(&errs, 1)
   243→				} else {
   244→					atomic.AddInt32(&n, 1)
   245→				}
   246→			}
   247→		}()
   248→	}
   249→
   250→	for i := 0; i < len(levels); i++ {
   251→		dir := path.Join(dir, path.Join(selectors[i]...))
   252→		work <- workItem{
   253→			level:    levels[i],
   254→			dir:      dir,
   255→			selector: selectors[i],
   256→		}
   257→	}
   258→
   259→	close(work)
   260→	wg.Wait()
   261→
   262→	if errs > 0 {
   263→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   264→	}
   265→	return int(n), nil
   266→}
   267→
   268→// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
   269→// Skips metrics that are already archived. Returns nil if no unarchived data exists.
   270→func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
   271→	l.lock.RLock()
   272→	defer l.lock.RUnlock()
   273→
   274→	retval := &CheckpointFile{
   275→		From:     from,
   276→		To:       to,
   277→		Metrics:  make(map[string]*CheckpointMetrics),
   278→		Children: make(map[string]*CheckpointFile),
   279→	}
   280→
   281→	for metric, minfo := range m.Metrics {
   282→		b := l.metrics[minfo.offset]
   283→		if b == nil {
   284→			continue
   285→		}
   286→
   287→		allArchived := true
   288→		b.iterFromTo(from, to, func(b *buffer) error {
   289→			if !b.archived {
   290→				allArchived = false
   291→				return fmt.Errorf("stop") // Early termination signal
   292→			}
   293→			return nil
   294→		})
   295→
   296→		if allArchived {
   297→			continue
   298→		}
   299→
   300→		data := make([]schema.Float, (to-from)/b.frequency+1)
   301→		data, start, end, err := b.read(from, to, data)
   302→		if err != nil {
   303→			return nil, err
   304→		}
   305→
   306→		for i := int((end - start) / b.frequency); i < len(data); i++ {
   307→			data[i] = schema.NaN
   308→		}
   309→
   310→		retval.Metrics[metric] = &CheckpointMetrics{
   311→			Frequency: b.frequency,
   312→			Start:     start,
   313→			Data:      data,
   314→		}
   315→	}
   316→
   317→	for name, child := range l.children {
   318→		val, err := child.toCheckpointFile(from, to, m)
   319→		if err != nil {
   320→			return nil, err
   321→		}
   322→
   323→		if val != nil {
   324→			retval.Children[name] = val
   325→		}
   326→	}
   327→
   328→	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
   329→		return nil, nil
   330→	}
   331→
   332→	return retval, nil
   333→}
   334→
   335→// toCheckpoint writes a Level's data to a JSON checkpoint file.
   336→// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.
   337→func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
   338→	cf, err := l.toCheckpointFile(from, to, m)
   339→	if err != nil {
   340→		return err
   341→	}
   342→
   343→	if cf == nil {
   344→		return ErrNoNewArchiveData
   345→	}
   346→
   347→	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
   348→	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   349→	if err != nil && os.IsNotExist(err) {
   350→		err = os.MkdirAll(dir, CheckpointDirPerms)
   351→		if err == nil {
   352→			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   353→		}
   354→	}
   355→	if err != nil {
   356→		return err
   357→	}
   358→	defer f.Close()
   359→
   360→	bw := bufio.NewWriter(f)
   361→	if err = json.NewEncoder(bw).Encode(cf); err != nil {
   362→		return err
   363→	}
   364→
   365→	return bw.Flush()
   366→}
   367→
   368→// enqueueCheckpointHosts traverses checkpoint directory and enqueues cluster/host pairs.
   369→// Returns error if directory structure is invalid.
   370→func enqueueCheckpointHosts(dir string, work chan<- [2]string) error {
   371→	clustersDir, err := os.ReadDir(dir)
   372→	if err != nil {
   373→		return err
   374→	}
   375→
   376→	for _, clusterDir := range clustersDir {
   377→		if !clusterDir.IsDir() {
   378→			return errors.New("[METRICSTORE]> expected only directories at first level of checkpoints/ directory")
   379→		}
   380→
   381→		hostsDir, err := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
   382→		if err != nil {
   383→			return err
   384→		}
   385→
   386→		for _, hostDir := range hostsDir {
   387→			if !hostDir.IsDir() {
   388→				return errors.New("[METRICSTORE]> expected only directories at second level of checkpoints/ directory")
   389→			}
   390→
   391→			work <- [2]string{clusterDir.Name(), hostDir.Name()}
   392→		}
   393→	}
   394→
   395→	return nil
   396→}
   397→
   398→// FromCheckpoint loads checkpoint files from disk into memory in parallel.
   399→//
   400→// Uses worker pool to load cluster/host combinations. Returns number of files
   401→// loaded and any errors.
   402→func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
   403→	var wg sync.WaitGroup
   404→	work := make(chan [2]string, Keys.NumWorkers*4)
   405→	n, errs := int32(0), int32(0)
   406→
   407→	wg.Add(Keys.NumWorkers)
   408→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   409→		go func() {
   410→			defer wg.Done()
   411→			for host := range work {
   412→				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
   413→				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
   414→				if err != nil {
   415→					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
   416→					atomic.AddInt32(&errs, 1)
   417→				}
   418→				atomic.AddInt32(&n, int32(nn))
   419→			}
   420→		}()
   421→	}
   422→
   423→	err := enqueueCheckpointHosts(dir, work)
   424→	close(work)
   425→	wg.Wait()
   426→
   427→	if err != nil {
   428→		return int(n), err
   429→	}
   430→
   431→	if errs > 0 {
   432→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   433→	}
   434→	return int(n), nil
   435→}
   436→
   437→// FromCheckpointFiles is the main entry point for loading checkpoints at startup.
   438→//
   439→// Creates checkpoint directory if it doesn't exist. This function must be called
   440→// before any writes or reads, and can only be called once.
   441→func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
   442→	if _, err := os.Stat(dir); os.IsNotExist(err) {
   443→		err := os.MkdirAll(dir, CheckpointDirPerms)
   444→		if err != nil {
   445→			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
   446→		}
   447→		cclog.Debugf("[METRICSTORE]> %#v Directory created successfully", dir)
   448→	}
   449→
   450→	return m.FromCheckpoint(dir, from)
   451→}
   452→
   453→func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
   454→	br := bufio.NewReader(f)
   455→	cf := &CheckpointFile{}
   456→	if err := json.NewDecoder(br).Decode(cf); err != nil {
   457→		return err
   458→	}
   459→
   460→	if cf.To != 0 && cf.To < from {
   461→		return nil
   462→	}
   463→
   464→	if err := l.loadFile(cf, m); err != nil {
   465→		return err
   466→	}
   467→
   468→	return nil
   469→}
   470→
   471→func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
   472→	for name, metric := range cf.Metrics {
   473→		n := len(metric.Data)
   474→		b := &buffer{
   475→			frequency: metric.Frequency,
   476→			start:     metric.Start,
   477→			data:      metric.Data[0:n:n],
   478→			prev:      nil,
   479→			next:      nil,
   480→			archived:  true,
   481→		}
   482→
   483→		minfo, ok := m.Metrics[name]
   484→		if !ok {
   485→			continue
   486→		}
   487→
   488→		prev := l.metrics[minfo.offset]
   489→		if prev == nil {
   490→			l.metrics[minfo.offset] = b
   491→		} else {
   492→			if prev.start > b.start {
   493→				return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start)
   494→			}
   495→
   496→			b.prev = prev
   497→			prev.next = b
   498→		}
   499→		l.metrics[minfo.offset] = b
   500→	}
   501→
   502→	if len(cf.Children) > 0 && l.children == nil {
   503→		l.children = make(map[string]*Level)
   504→	}
   505→
   506→	for sel, childCf := range cf.Children {
   507→		child, ok := l.children[sel]
   508→		if !ok {
   509→			child = &Level{
   510→				metrics:  make([]*buffer, len(m.Metrics)),
   511→				children: nil,
   512→			}
   513→			l.children[sel] = child
   514→		}
   515→
   516→		if err := child.loadFile(childCf, m); err != nil {
   517→			return err
   518→		}
   519→	}
   520→
   521→	return nil
   522→}
   523→
   524→// fromCheckpoint loads all checkpoint files (JSON, binary snapshot, WAL) for a
   525→// single host directory. Snapshot files are loaded first (sorted by timestamp),
   526→// then current.wal is replayed on top.
   527→func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {
   528→	direntries, err := os.ReadDir(dir)
   529→	if err != nil {
   530→		if os.IsNotExist(err) {
   531→			return 0, nil
   532→		}
   533→		return 0, err
   534→	}
   535→
   536→	allFiles := make([]fs.DirEntry, 0)
   537→	var walEntry fs.DirEntry
   538→	filesLoaded := 0
   539→
   540→	for _, e := range direntries {
   541→		if e.IsDir() {
   542→			// Legacy: skip subdirectories (only used by old Avro format).
   543→			// These are ignored; their data is not loaded.
   544→			cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir)
   545→			continue
   546→		}
   547→
   548→		name := e.Name()
   549→		if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") {
   550→			allFiles = append(allFiles, e)
   551→		} else if name == "current.wal" {
   552→			walEntry = e
   553→		}
   554→		// Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes).
   555→	}
   556→
   557→	files, err := findFiles(allFiles, from, true)
   558→	if err != nil {
   559→		return filesLoaded, err
   560→	}
   561→
   562→	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
   563→		".json": l.loadJSONFile,
   564→		".bin":  l.loadBinaryFile,
   565→	}
   566→
   567→	for _, filename := range files {
   568→		ext := filepath.Ext(filename)
   569→		loader := loaders[ext]
   570→		if loader == nil {
   571→			cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename)
   572→			continue
   573→		}
   574→
   575→		err := func() error {
   576→			f, err := os.Open(path.Join(dir, filename))
   577→			if err != nil {
   578→				return err
   579→			}
   580→			defer f.Close()
   581→			return loader(m, f, from)
   582→		}()
   583→		if err != nil {
   584→			return filesLoaded, err
   585→		}
   586→		filesLoaded++
   587→	}
   588→
   589→	// Replay WAL after all snapshot files so it fills in data since the last snapshot.
   590→	if walEntry != nil {
   591→		err := func() error {
   592→			f, err := os.Open(path.Join(dir, walEntry.Name()))
   593→			if err != nil {
   594→				return err
   595→			}
   596→			defer f.Close()
   597→			return l.loadWALFile(m, f, from)
   598→		}()
   599→		if err != nil {
   600→			// WAL errors are non-fatal: the snapshot already loaded the bulk of data.
   601→			cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err)
   602→		} else {
   603→			filesLoaded++
   604→		}
   605→	}
   606→
   607→	return filesLoaded, nil
   608→}
   609→
   610→// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename.
   611→// Supports ".json" (format: "<ts>.json") and ".bin" (format: "<ts>.bin").
   612→func parseTimestampFromFilename(name string) (int64, error) {
   613→	switch {
   614→	case strings.HasSuffix(name, ".json"):
   615→		return strconv.ParseInt(name[:len(name)-5], 10, 64)
   616→	case strings.HasSuffix(name, ".bin"):
   617→		return strconv.ParseInt(name[:len(name)-4], 10, 64)
   618→	default:
   619→		return 0, fmt.Errorf("unknown checkpoint extension for file %q", name)
   620→	}
   621→}
   622→
   623→// findFiles returns filenames from direntries whose timestamps satisfy the filter.
   624→// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the
   625→// last file before t if t falls between two files).
   626→func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {
   627→	nums := map[string]int64{}
   628→	for _, e := range direntries {
   629→		name := e.Name()
   630→		if !strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".bin") {
   631→			continue
   632→		}
   633→
   634→		ts, err := parseTimestampFromFilename(name)
   635→		if err != nil {
   636→			return nil, err
   637→		}
   638→		nums[name] = ts
   639→	}
   640→
   641→	sort.Slice(direntries, func(i, j int) bool {
   642→		a, b := direntries[i], direntries[j]
   643→		return nums[a.Name()] < nums[b.Name()]
   644→	})
   645→
   646→	if len(nums) == 0 {
   647→		return nil, nil
   648→	}
   649→
   650→	filenames := make([]string, 0)
   651→
   652→	for i, e := range direntries {
   653→		ts1 := nums[e.Name()]
   654→
   655→		if findMoreRecentFiles && t <= ts1 {
   656→			filenames = append(filenames, e.Name())
   657→		} else if !findMoreRecentFiles && ts1 <= t && ts1 != 0 {
   658→			filenames = append(filenames, e.Name())
   659→		}
   660→
   661→		if i == len(direntries)-1 {
   662→			continue
   663→		}
   664→
   665→		enext := direntries[i+1]
   666→		ts2 := nums[enext.Name()]
   667→
   668→		if findMoreRecentFiles {
   669→			if ts1 < t && t < ts2 {
   670→				filenames = append(filenames, e.Name())
   671→			}
   672→		}
   673→	}
   674→
   675→	return filenames, nil
   676→}
   677→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements checkpoint persistence for the in-memory metric store.
//
// Checkpoints enable graceful restarts by periodically saving in-memory metric
// data to disk in JSON or binary format. The checkpoint system:
//
// Key Features:
//   - Periodic background checkpointing via the Checkpointing() worker
//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
//   - Parallel checkpoint creation and loading using worker pools
//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
//   - Only saves unarchived data (archived data is already persisted elsewhere)
//   - GC optimization during loading to prevent excessive heap growth
//
// Checkpoint Workflow:
//  1. Init() loads checkpoints within retention window at startup
//  2. Checkpointing() worker periodically saves new data
//  3. Shutdown() writes final checkpoint before exit
//
// File Organization:
//
//	checkpoints/
//	  cluster1/
//	    host001/
//	      1234567890.json  (JSON format: full subtree snapshot)
//	      1234567890.bin   (binary format: full subtree snapshot)
//	      current.wal      (WAL format: append-only per-entry log)
//	    host002/
//	      ...
package metricstore

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

const (
	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
)

// CheckpointMetrics represents metric data in a checkpoint file.
// Whenever the structure changes, update MarshalJSON as well!
type CheckpointMetrics struct {
	Data      []schema.Float `json:"data"`
	Frequency int64          `json:"frequency"`
	Start     int64          `json:"start"`
}

// CheckpointFile represents the hierarchical structure of a checkpoint file.
// It mirrors the Level tree structure from the MemoryStore.
type CheckpointFile struct {
	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
	Children map[string]*CheckpointFile    `json:"children"`
	From     int64                         `json:"from"`
	To       int64                         `json:"to"`
}

// lastCheckpoint tracks the timestamp of the last checkpoint creation.
var (
	lastCheckpoint   time.Time
	lastCheckpointMu sync.Mutex
)

// Checkpointing starts a background worker that periodically saves metric data to disk.
//
// Checkpoints are written every 12 hours (hardcoded).
//
// Format behaviour:
//   - "json": Periodic checkpointing every checkpointInterval
//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
	lastCheckpointMu.Lock()
	lastCheckpoint = time.Now()
	lastCheckpointMu.Unlock()

	ms := GetMemoryStore()

	wg.Go(func() {

		d := 12 * time.Hour // default checkpoint interval
		if Keys.CheckpointInterval != "" {
			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
			if err != nil {
				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
			} else {
				d = parsed
			}
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				lastCheckpointMu.Lock()
				from := lastCheckpoint
				lastCheckpointMu.Unlock()

				now := time.Now()
				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))

				if Keys.Checkpoints.FileFormat == "wal" {
					// Pause WAL writes: the binary snapshot captures all in-memory
					// data, so WAL records written during checkpoint are redundant
					// and would be deleted during rotation anyway.
					walCheckpointActive.Store(true)
					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
					}
					if n > 0 {
						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
						// Rotate WAL files for successfully checkpointed hosts.
						RotateWALFiles(hostDirs)
					}
					walCheckpointActive.Store(false)
					walDropped.Store(0)
				} else {
					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
					}
				}
			}
		}
	})
}

// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
//
// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
// This method manually constructs JSON to avoid allocations and interface conversions.
func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
	buf := make([]byte, 0, 128+len(cm.Data)*8)
	buf = append(buf, `{"frequency":`...)
	buf = strconv.AppendInt(buf, cm.Frequency, 10)
	buf = append(buf, `,"start":`...)
	buf = strconv.AppendInt(buf, cm.Start, 10)
	buf = append(buf, `,"data":[`...)
	for i, x := range cm.Data {
		if i != 0 {
			buf = append(buf, ',')
		}
		if x.IsNaN() {
			buf = append(buf, `null`...)
		} else {
			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
		}
	}
	buf = append(buf, `]}`...)
	return buf, nil
}

// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format).
//
// Metrics at root and cluster levels are skipped. One file per host is created.
// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
// at a time, allowing concurrent writes/reads to other hosts.
//
// Returns the number of checkpoint files created and any errors encountered.
func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
	// Pre-calculate capacity by counting cluster/host pairs
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		dir      string
		selector []string
	}

	n, errs := int32(0), int32(0)

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()

			for workItem := range work {
				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
					if err == ErrNoNewArchiveData {
						continue
					}

					cclog.Errorf("[METRICSTORE]> error while checkpointing %#v: %s", workItem.selector, err.Error())
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
				}
			}
		}()
	}

	for i := 0; i < len(levels); i++ {
		dir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			dir:      dir,
			selector: selectors[i],
		}
	}

	close(work)
	wg.Wait()

	if errs > 0 {
		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
	}
	return int(n), nil
}

// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
// Skips metrics that are already archived. Returns nil if no unarchived data exists.
func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	retval := &CheckpointFile{
		From:     from,
		To:       to,
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	for metric, minfo := range m.Metrics {
		b := l.metrics[minfo.offset]
		if b == nil {
			continue
		}

		allArchived := true
		b.iterFromTo(from, to, func(b *buffer) error {
			if !b.archived {
				allArchived = false
				return fmt.Errorf("stop") // Early termination signal
			}
			return nil
		})

		if allArchived {
			continue
		}

		data := make([]schema.Float, (to-from)/b.frequency+1)
		data, start, end, err := b.read(from, to, data)
		if err != nil {
			return nil, err
		}

		for i := int((end - start) / b.frequency); i < len(data); i++ {
			data[i] = schema.NaN
		}

		retval.Metrics[metric] = &CheckpointMetrics{
			Frequency: b.frequency,
			Start:     start,
			Data:      data,
		}
	}

	for name, child := range l.children {
		val, err := child.toCheckpointFile(from, to, m)
		if err != nil {
			return nil, err
		}

		if val != nil {
			retval.Children[name] = val
		}
	}

	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
		return nil, nil
	}

	return retval, nil
}

// toCheckpoint writes a Level's data to a JSON checkpoint file.
// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.
func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}

	if cf == nil {
		return ErrNoNewArchiveData
	}

	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil && os.IsNotExist(err) {
		err = os.MkdirAll(dir, CheckpointDirPerms)
		if err == nil {
			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
		}
	}
	if err != nil {
		return err
	}
	defer f.Close()

	bw := bufio.NewWriter(f)
	if err = json.NewEncoder(bw).Encode(cf); err != nil {
		return err
	}

	return bw.Flush()
}

// enqueueCheckpointHosts traverses checkpoint directory and enqueues cluster/host pairs.
// Returns error if directory structure is invalid.
func enqueueCheckpointHosts(dir string, work chan<- [2]string) error {
	clustersDir, err := os.ReadDir(dir)
	if err != nil {
		return err
	}

	for _, clusterDir := range clustersDir {
		if !clusterDir.IsDir() {
			return errors.New("[METRICSTORE]> expected only directories at first level of checkpoints/ directory")
		}

		hostsDir, err := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
		if err != nil {
			return err
		}

		for _, hostDir := range hostsDir {
			if !hostDir.IsDir() {
				return errors.New("[METRICSTORE]> expected only directories at second level of checkpoints/ directory")
			}

			work <- [2]string{clusterDir.Name(), hostDir.Name()}
		}
	}

	return nil
}

// FromCheckpoint loads checkpoint files from disk into memory in parallel.
//
// Uses worker pool to load cluster/host combinations. Returns number of files
// loaded and any errors.
func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
	var wg sync.WaitGroup
	work := make(chan [2]string, Keys.NumWorkers*4)
	n, errs := int32(0), int32(0)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for host := range work {
				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
					atomic.AddInt32(&errs, 1)
				}
				atomic.AddInt32(&n, int32(nn))
			}
		}()
	}

	err := enqueueCheckpointHosts(dir, work)
	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}

	if errs > 0 {
		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
	}
	return int(n), nil
}

// FromCheckpointFiles is the main entry point for loading checkpoints at startup.
//
// Creates checkpoint directory if it doesn't exist. This function must be called
// before any writes or reads, and can only be called once.
func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, CheckpointDirPerms)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
		}
		cclog.Debugf("[METRICSTORE]> %#v Directory created successfully", dir)
	}

	return m.FromCheckpoint(dir, from)
}

func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)
	cf := &CheckpointFile{}
	if err := json.NewDecoder(br).Decode(cf); err != nil {
		return err
	}

	if cf.To != 0 && cf.To < from {
		return nil
	}

	if err := l.loadFile(cf, m); err != nil {
		return err
	}

	return nil
}

func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
	for name, metric := range cf.Metrics {
		n := len(metric.Data)
		b := &buffer{
			frequency: metric.Frequency,
			start:     metric.Start,
			data:      metric.Data[0:n:n],
			prev:      nil,
			next:      nil,
			archived:  true,
		}

		minfo, ok := m.Metrics[name]
		if !ok {
			continue
		}

		prev := l.metrics[minfo.offset]
		if prev == nil {
			l.metrics[minfo.offset] = b
		} else {
			if prev.start > b.start {
				return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start)
			}

			b.prev = prev
			prev.next = b
		}
		l.metrics[minfo.offset] = b
	}

	if len(cf.Children) > 0 && l.children == nil {
		l.children = make(map[string]*Level)
	}

	for sel, childCf := range cf.Children {
		child, ok := l.children[sel]
		if !ok {
			child = &Level{
				metrics:  make([]*buffer, len(m.Metrics)),
				children: nil,
			}
			l.children[sel] = child
		}

		if err := child.loadFile(childCf, m); err != nil {
			return err
		}
	}

	return nil
}

// fromCheckpoint loads all checkpoint files (JSON, binary snapshot, WAL) for a
// single host directory. Snapshot files are loaded first (sorted by timestamp),
// then current.wal is replayed on top.
func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {
	direntries, err := os.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return 0, nil
		}
		return 0, err
	}

	allFiles := make([]fs.DirEntry, 0)
	var walEntry fs.DirEntry
	filesLoaded := 0

	for _, e := range direntries {
		if e.IsDir() {
			// Legacy: skip subdirectories (only used by old Avro format).
			// These are ignored; their data is not loaded.
			cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir)
			continue
		}

		name := e.Name()
		if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") {
			allFiles = append(allFiles, e)
		} else if name == "current.wal" {
			walEntry = e
		}
		// Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes).
	}

	files, err := findFiles(allFiles, from, true)
	if err != nil {
		return filesLoaded, err
	}

	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
		".json": l.loadJSONFile,
		".bin":  l.loadBinaryFile,
	}

	for _, filename := range files {
		ext := filepath.Ext(filename)
		loader := loaders[ext]
		if loader == nil {
			cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename)
			continue
		}

		err := func() error {
			f, err := os.Open(path.Join(dir, filename))
			if err != nil {
				return err
			}
			defer f.Close()
			return loader(m, f, from)
		}()
		if err != nil {
			return filesLoaded, err
		}
		filesLoaded++
	}

	// Replay WAL after all snapshot files so it fills in data since the last snapshot.
	if walEntry != nil {
		err := func() error {
			f, err := os.Open(path.Join(dir, walEntry.Name()))
			if err != nil {
				return err
			}
			defer f.Close()
			return l.loadWALFile(m, f, from)
		}()
		if err != nil {
			// WAL errors are non-fatal: the snapshot already loaded the bulk of data.
			cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err)
		} else {
			filesLoaded++
		}
	}

	return filesLoaded, nil
}

// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename.
// Supports ".json" (format: "<ts>.json") and ".bin" (format: "<ts>.bin").
func parseTimestampFromFilename(name string) (int64, error) {
	switch {
	case strings.HasSuffix(name, ".json"):
		return strconv.ParseInt(name[:len(name)-5], 10, 64)
	case strings.HasSuffix(name, ".bin"):
		return strconv.ParseInt(name[:len(name)-4], 10, 64)
	default:
		return 0, fmt.Errorf("unknown checkpoint extension for file %q", name)
	}
}

// findFiles returns filenames from direntries whose timestamps satisfy the filter.
// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the
// last file before t if t falls between two files).
func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {
	nums := map[string]int64{}
	for _, e := range direntries {
		name := e.Name()
		if !strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".bin") {
			continue
		}

		ts, err := parseTimestampFromFilename(name)
		if err != nil {
			return nil, err
		}
		nums[name] = ts
	}

	sort.Slice(direntries, func(i, j int) bool {
		a, b := direntries[i], direntries[j]
		return nums[a.Name()] < nums[b.Name()]
	})

	if len(nums) == 0 {
		return nil, nil
	}

	filenames := make([]string, 0)

	for i, e := range direntries {
		ts1 := nums[e.Name()]

		if findMoreRecentFiles && t <= ts1 {
			filenames = append(filenames, e.Name())
		} else if !findMoreRecentFiles && ts1 <= t && ts1 != 0 {
			filenames = append(filenames, e.Name())
		}

		if i == len(direntries)-1 {
			continue
		}

		enext := direntries[i+1]
		ts2 := nums[enext.Name()]

		if findMoreRecentFiles {
			if ts1 < t && t < ts2 {
				filenames = append(filenames, e.Name())
			}
		}
	}

	return filenames, nil
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides level.go: Hierarchical tree structure for metric storage.
     7→//
     8→// # Level Architecture
     9→//
    10→// The Level type forms a tree structure where each node represents a level in the
    11→// ClusterCockpit hierarchy: cluster → host → socket → core → hwthread, with special
    12→// nodes for memory domains and accelerators.
    13→//
    14→// Structure:
    15→//
    16→//	Root Level (cluster="emmy")
    17→//	├─ Level (host="node001")
    18→//	│  ├─ Level (socket="0")
    19→//	│  │  ├─ Level (core="0") [stores cpu0 metrics]
    20→//	│  │  └─ Level (core="1") [stores cpu1 metrics]
    21→//	│  └─ Level (socket="1")
    22→//	│     └─ ...
    23→//	└─ Level (host="node002")
    24→//	   └─ ...
    25→//
    26→// Each Level can:
    27→//   - Hold data (metrics slice of buffer pointers)
    28→//   - Have child nodes (children map[string]*Level)
    29→//   - Both simultaneously (inner nodes can store aggregated metrics)
    30→//
    31→// # Selector Paths
    32→//
    33→// Selectors are hierarchical paths: []string{"cluster", "host", "component"}.
    34→// Example: []string{"emmy", "node001", "cpu0"} navigates to the cpu0 core level.
    35→//
    36→// # Concurrency
    37→//
    38→// RWMutex protects children map and metrics slice. Read-heavy workload (metric reads)
    39→// uses RLock. Writes (new levels, buffer updates) use Lock. Double-checked locking
    40→// prevents races during level creation.
    41→package metricstore
    42→
    43→import (
    44→	"sync"
    45→	"time"
    46→	"unsafe"
    47→
    48→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    49→	"github.com/ClusterCockpit/cc-lib/v2/util"
    50→)
    51→
    52→// Level represents a node in the hierarchical metric storage tree.
    53→//
    54→// Can be both a leaf or inner node. Inner nodes hold data in 'metrics' for aggregated
    55→// values (e.g., socket-level metrics derived from core-level data). Named "Level"
    56→// instead of "node" to avoid confusion with cluster nodes (hosts).
    57→//
    58→// Fields:
    59→//   - children: Map of child level names to Level pointers (e.g., "cpu0" → Level)
    60→//   - metrics:  Slice of buffer pointers (one per metric, indexed by MetricConfig.offset)
    61→//   - lock:     RWMutex for concurrent access (read-heavy, write-rare)
    62→type Level struct {
    63→	children map[string]*Level
    64→	metrics  []*buffer
    65→	lock     sync.RWMutex
    66→}
    67→
    68→// findLevelOrCreate navigates to or creates the level specified by selector.
    69→//
    70→// Recursively descends the tree, creating missing levels as needed. Uses double-checked
    71→// locking: RLock first (fast path), then Lock if creation needed (slow path), then
    72→// re-check after acquiring Lock to handle races.
    73→//
    74→// Example selector: []string{"emmy", "node001", "cpu0"}
    75→// Navigates: root → emmy → node001 → cpu0, creating levels as needed.
    76→//
    77→// Parameters:
    78→//   - selector: Hierarchical path (consumed recursively, decreasing depth)
    79→//   - nMetrics: Number of metric slots to allocate in new levels
    80→//
    81→// Returns:
    82→//   - *Level: The target level (existing or newly created)
    83→//
    84→// Note: sync.Map may improve performance for high-concurrency writes, but current
    85→// approach suffices for read-heavy workload.
    86→func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
    87→	if len(selector) == 0 {
    88→		return l
    89→	}
    90→
    91→	// Allow concurrent reads:
    92→	l.lock.RLock()
    93→	var child *Level
    94→	var ok bool
    95→	if l.children == nil {
    96→		// Children map needs to be created...
    97→		l.lock.RUnlock()
    98→	} else {
    99→		child, ok = l.children[selector[0]]
   100→		l.lock.RUnlock()
   101→		if ok {
   102→			return child.findLevelOrCreate(selector[1:], nMetrics)
   103→		}
   104→	}
   105→
   106→	// The level does not exist, take write lock for unique access:
   107→	l.lock.Lock()
   108→	// While this thread waited for the write lock, another thread
   109→	// could have created the child node.
   110→	if l.children != nil {
   111→		child, ok = l.children[selector[0]]
   112→		if ok {
   113→			l.lock.Unlock()
   114→			return child.findLevelOrCreate(selector[1:], nMetrics)
   115→		}
   116→	}
   117→
   118→	child = &Level{
   119→		metrics:  make([]*buffer, nMetrics),
   120→		children: nil,
   121→	}
   122→
   123→	if l.children != nil {
   124→		l.children[selector[0]] = child
   125→	} else {
   126→		l.children = map[string]*Level{selector[0]: child}
   127→	}
   128→	l.lock.Unlock()
   129→	return child.findLevelOrCreate(selector[1:], nMetrics)
   130→}
   131→
   132→// collectPaths gathers all selector paths at the specified depth in the tree.
   133→//
   134→// Recursively traverses children, collecting paths when currentDepth+1 == targetDepth.
   135→// Each path is a selector that can be used with findLevel() or findBuffers().
   136→//
   137→// Explicitly copies slices to avoid shared underlying arrays between siblings, preventing
   138→// unintended mutations.
   139→//
   140→// Parameters:
   141→//   - currentDepth: Depth of current level (0 = root)
   142→//   - targetDepth:  Depth to collect paths from
   143→//   - currentPath:  Path accumulated so far
   144→//   - results:      Output slice (appended to)
   145→//
   146→// Example: collectPaths(0, 2, []string{}, &results) collects all 2-level paths
   147→// like []string{"emmy", "node001"}, []string{"emmy", "node002"}, etc.
   148→func (l *Level) collectPaths(currentDepth, targetDepth int, currentPath []string, results *[][]string) {
   149→	l.lock.RLock()
   150→	defer l.lock.RUnlock()
   151→
   152→	for key, child := range l.children {
   153→		if child == nil {
   154→			continue
   155→		}
   156→
   157→		// We explicitly make a new slice and copy data to avoid sharing underlying arrays between siblings
   158→		newPath := make([]string, len(currentPath))
   159→		copy(newPath, currentPath)
   160→		newPath = append(newPath, key)
   161→
   162→		// Check depth, and just return if depth reached
   163→		if currentDepth+1 == targetDepth {
   164→			*results = append(*results, newPath)
   165→		} else {
   166→			child.collectPaths(currentDepth+1, targetDepth, newPath, results)
   167→		}
   168→	}
   169→}
   170→
   171→// free removes buffers older than the retention threshold from the entire subtree.
   172→//
   173→// Recursively frees buffers in this level's metrics and all child levels. Buffers
   174→// with standard capacity (BufferCap) are returned to the pool. Called by the
   175→// retention worker to enforce retention policies.
   176→//
   177→// Parameters:
   178→//   - t: Retention threshold timestamp (Unix seconds)
   179→//
   180→// Returns:
   181→//   - int:   Total number of buffers freed in this subtree
   182→//   - error: Non-nil on failure (propagated from children)
   183→func (l *Level) free(t int64) (int, error) {
   184→	n, _, err := l.freeAndCheckEmpty(t)
   185→	return n, err
   186→}
   187→
   188→// freeAndCheckEmpty performs free() and atomically checks if the level is empty
   189→// while still holding its own lock, avoiding a TOCTOU race between free() and
   190→// a separate isEmpty() call.
   191→func (l *Level) freeAndCheckEmpty(t int64) (int, bool, error) {
   192→	l.lock.Lock()
   193→	defer l.lock.Unlock()
   194→
   195→	n := 0
   196→	for i, b := range l.metrics {
   197→		if b != nil {
   198→			delme, m := b.free(t)
   199→			n += m
   200→			if delme {
   201→				if cap(b.data) != BufferCap {
   202→					b.data = make([]schema.Float, 0, BufferCap)
   203→				}
   204→				b.lastUsed = time.Now().Unix()
   205→				bufferPool.Put(b)
   206→				l.metrics[i] = nil
   207→			}
   208→		}
   209→	}
   210→
   211→	for key, child := range l.children {
   212→		m, empty, err := child.freeAndCheckEmpty(t)
   213→		n += m
   214→		if err != nil {
   215→			return n, false, err
   216→		}
   217→		if empty {
   218→			delete(l.children, key)
   219→		}
   220→	}
   221→
   222→	// Check emptiness while still holding the lock
   223→	empty := len(l.children) == 0
   224→	if empty {
   225→		for _, b := range l.metrics {
   226→			if b != nil {
   227→				empty = false
   228→				break
   229→			}
   230→		}
   231→	}
   232→
   233→	return n, empty, nil
   234→}
   235→
   236→// forceFree removes the oldest buffer from each metric chain in the subtree.
   237→//
   238→// Unlike free(), which removes based on time threshold, this unconditionally removes
   239→// the oldest buffer in each chain. Used by MemoryUsageTracker when memory cap is
   240→// exceeded and time-based retention is insufficient.
   241→//
   242→// Recursively processes current level's metrics and all child levels.
   243→//
   244→// Returns:
   245→//   - int:   Total number of buffers freed in this subtree
   246→//   - error: Non-nil on failure (propagated from children)
   247→func (l *Level) forceFree() (int, error) {
   248→	l.lock.Lock()
   249→	defer l.lock.Unlock()
   250→
   251→	n := 0
   252→
   253→	// Iterate over metrics in the current level
   254→	for i, b := range l.metrics {
   255→		if b != nil {
   256→			// Attempt to free the oldest buffer in this chain
   257→			delme, freedCount := b.forceFreeOldest()
   258→			n += freedCount
   259→
   260→			// If delme is true, it means 'b' itself (the head) was the oldest
   261→			// and needs to be removed from the slice.
   262→			if delme {
   263→				b.next = nil
   264→				b.prev = nil
   265→				if cap(b.data) != BufferCap {
   266→					b.data = make([]schema.Float, 0, BufferCap)
   267→				}
   268→				b.lastUsed = time.Now().Unix()
   269→				bufferPool.Put(b)
   270→				l.metrics[i] = nil
   271→			}
   272→		}
   273→	}
   274→
   275→	// Recursively traverse children
   276→	for _, child := range l.children {
   277→		m, err := child.forceFree()
   278→		n += m
   279→		if err != nil {
   280→			return n, err
   281→		}
   282→	}
   283→
   284→	return n, nil
   285→}
   286→
   287→// sizeInBytes calculates the total memory usage of all buffers in the subtree.
   288→//
   289→// Recursively sums buffer data sizes (count of Float values × sizeof(Float)) across
   290→// this level's metrics and all child levels. Used by MemoryUsageTracker to enforce
   291→// memory cap limits.
   292→//
   293→// Returns:
   294→//   - int64: Total bytes used by buffer data in this subtree
   295→func (l *Level) sizeInBytes() int64 {
   296→	l.lock.RLock()
   297→	defer l.lock.RUnlock()
   298→	size := int64(0)
   299→
   300→	for _, b := range l.metrics {
   301→		if b != nil {
   302→			size += b.count() * int64(unsafe.Sizeof(schema.Float(0)))
   303→			size += b.bufferCount() * int64(unsafe.Sizeof(buffer{}))
   304→		}
   305→	}
   306→
   307→	for _, child := range l.children {
   308→		size += child.sizeInBytes()
   309→	}
   310→
   311→	return size
   312→}
   313→
   314→// findLevel navigates to the level specified by selector, returning nil if not found.
   315→//
   316→// Read-only variant of findLevelOrCreate. Does not create missing levels.
   317→// Recursively descends the tree following the selector path.
   318→//
   319→// Parameters:
   320→//   - selector: Hierarchical path (e.g., []string{"emmy", "node001", "cpu0"})
   321→//
   322→// Returns:
   323→//   - *Level: The target level, or nil if any component in the path does not exist
   324→func (l *Level) findLevel(selector []string) *Level {
   325→	if len(selector) == 0 {
   326→		return l
   327→	}
   328→
   329→	l.lock.RLock()
   330→	defer l.lock.RUnlock()
   331→
   332→	lvl := l.children[selector[0]]
   333→	if lvl == nil {
   334→		return nil
   335→	}
   336→
   337→	return lvl.findLevel(selector[1:])
   338→}
   339→
   340→// findBuffers invokes callback on all buffers matching the selector pattern.
   341→//
   342→// Supports flexible selector patterns (from cc-lib/util.Selector):
   343→//   - Exact match: Selector element with String set (e.g., "node001")
   344→//   - Group match: Selector element with Group set (e.g., ["cpu0", "cpu2", "cpu4"])
   345→//   - Wildcard:    Selector element with Any=true (matches all children)
   346→//
   347→// Empty selector (len==0) matches current level's buffer at 'offset' and recursively
   348→// all descendant buffers at the same offset (used for aggregation queries).
   349→//
   350→// Parameters:
   351→//   - selector: Pattern to match (consumed recursively)
   352→//   - offset:   Metric index in metrics slice (from MetricConfig.offset)
   353→//   - f:        Callback invoked on each matching buffer
   354→//
   355→// Returns:
   356→//   - error: First error returned by callback, or nil if all succeeded
   357→//
   358→// Example:
   359→//
   360→//	// Find all cpu0 buffers across all hosts:
   361→//	findBuffers([]Selector{{Any: true}, {String: "cpu0"}}, metricOffset, callback)
   362→func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
   363→	l.lock.RLock()
   364→	defer l.lock.RUnlock()
   365→
   366→	if len(selector) == 0 {
   367→		b := l.metrics[offset]
   368→		if b != nil {
   369→			return f(b)
   370→		}
   371→
   372→		for _, lvl := range l.children {
   373→			err := lvl.findBuffers(nil, offset, f)
   374→			if err != nil {
   375→				return err
   376→			}
   377→		}
   378→		return nil
   379→	}
   380→
   381→	sel := selector[0]
   382→	if len(sel.String) != 0 && l.children != nil {
   383→		lvl, ok := l.children[sel.String]
   384→		if ok {
   385→			err := lvl.findBuffers(selector[1:], offset, f)
   386→			if err != nil {
   387→				return err
   388→			}
   389→		}
   390→		return nil
   391→	}
   392→
   393→	if sel.Group != nil && l.children != nil {
   394→		for _, key := range sel.Group {
   395→			lvl, ok := l.children[key]
   396→			if ok {
   397→				err := lvl.findBuffers(selector[1:], offset, f)
   398→				if err != nil {
   399→					return err
   400→				}
   401→			}
   402→		}
   403→		return nil
   404→	}
   405→
   406→	if sel.Any && l.children != nil {
   407→		for _, lvl := range l.children {
   408→			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
   409→				return err
   410→			}
   411→		}
   412→		return nil
   413→	}
   414→
   415→	return nil
   416→}
   417→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides level.go: Hierarchical tree structure for metric storage.
//
// # Level Architecture
//
// The Level type forms a tree structure where each node represents a level in the
// ClusterCockpit hierarchy: cluster → host → socket → core → hwthread, with special
// nodes for memory domains and accelerators.
//
// Structure:
//
//	Root Level (cluster="emmy")
//	├─ Level (host="node001")
//	│  ├─ Level (socket="0")
//	│  │  ├─ Level (core="0") [stores cpu0 metrics]
//	│  │  └─ Level (core="1") [stores cpu1 metrics]
//	│  └─ Level (socket="1")
//	│     └─ ...
//	└─ Level (host="node002")
//	   └─ ...
//
// Each Level can:
//   - Hold data (metrics slice of buffer pointers)
//   - Have child nodes (children map[string]*Level)
//   - Both simultaneously (inner nodes can store aggregated metrics)
//
// # Selector Paths
//
// Selectors are hierarchical paths: []string{"cluster", "host", "component"}.
// Example: []string{"emmy", "node001", "cpu0"} navigates to the cpu0 core level.
//
// # Concurrency
//
// RWMutex protects children map and metrics slice. Read-heavy workload (metric reads)
// uses RLock. Writes (new levels, buffer updates) use Lock. Double-checked locking
// prevents races during level creation.
package metricstore

import (
	"sync"
	"time"
	"unsafe"

	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-lib/v2/util"
)

// Level represents a node in the hierarchical metric storage tree.
//
// Can be both a leaf or inner node. Inner nodes hold data in 'metrics' for aggregated
// values (e.g., socket-level metrics derived from core-level data). Named "Level"
// instead of "node" to avoid confusion with cluster nodes (hosts).
//
// Fields:
//   - children: Map of child level names to Level pointers (e.g., "cpu0" → Level)
//   - metrics:  Slice of buffer pointers (one per metric, indexed by MetricConfig.offset)
//   - lock:     RWMutex for concurrent access (read-heavy, write-rare)
type Level struct {
	children map[string]*Level
	metrics  []*buffer
	lock     sync.RWMutex
}

// findLevelOrCreate navigates to or creates the level specified by selector.
//
// Recursively descends the tree, creating missing levels as needed. Uses double-checked
// locking: RLock first (fast path), then Lock if creation needed (slow path), then
// re-check after acquiring Lock to handle races.
//
// Example selector: []string{"emmy", "node001", "cpu0"}
// Navigates: root → emmy → node001 → cpu0, creating levels as needed.
//
// Parameters:
//   - selector: Hierarchical path (consumed recursively, decreasing depth)
//   - nMetrics: Number of metric slots to allocate in new levels
//
// Returns:
//   - *Level: The target level (existing or newly created)
//
// Note: sync.Map may improve performance for high-concurrency writes, but current
// approach suffices for read-heavy workload.
func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
	if len(selector) == 0 {
		return l
	}

	// Allow concurrent reads:
	l.lock.RLock()
	var child *Level
	var ok bool
	if l.children == nil {
		// Children map needs to be created...
		l.lock.RUnlock()
	} else {
		child, ok = l.children[selector[0]]
		l.lock.RUnlock()
		if ok {
			return child.findLevelOrCreate(selector[1:], nMetrics)
		}
	}

	// The level does not exist, take write lock for unique access:
	l.lock.Lock()
	// While this thread waited for the write lock, another thread
	// could have created the child node.
	if l.children != nil {
		child, ok = l.children[selector[0]]
		if ok {
			l.lock.Unlock()
			return child.findLevelOrCreate(selector[1:], nMetrics)
		}
	}

	child = &Level{
		metrics:  make([]*buffer, nMetrics),
		children: nil,
	}

	if l.children != nil {
		l.children[selector[0]] = child
	} else {
		l.children = map[string]*Level{selector[0]: child}
	}
	l.lock.Unlock()
	return child.findLevelOrCreate(selector[1:], nMetrics)
}

// collectPaths gathers all selector paths at the specified depth in the tree.
//
// Recursively traverses children, collecting paths when currentDepth+1 == targetDepth.
// Each path is a selector that can be used with findLevel() or findBuffers().
//
// Explicitly copies slices to avoid shared underlying arrays between siblings, preventing
// unintended mutations.
//
// Parameters:
//   - currentDepth: Depth of current level (0 = root)
//   - targetDepth:  Depth to collect paths from
//   - currentPath:  Path accumulated so far
//   - results:      Output slice (appended to)
//
// Example: collectPaths(0, 2, []string{}, &results) collects all 2-level paths
// like []string{"emmy", "node001"}, []string{"emmy", "node002"}, etc.
func (l *Level) collectPaths(currentDepth, targetDepth int, currentPath []string, results *[][]string) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	for key, child := range l.children {
		if child == nil {
			continue
		}

		// We explicitly make a new slice and copy data to avoid sharing underlying arrays between siblings
		newPath := make([]string, len(currentPath))
		copy(newPath, currentPath)
		newPath = append(newPath, key)

		// Check depth, and just return if depth reached
		if currentDepth+1 == targetDepth {
			*results = append(*results, newPath)
		} else {
			child.collectPaths(currentDepth+1, targetDepth, newPath, results)
		}
	}
}

// free removes buffers older than the retention threshold from the entire subtree.
//
// Recursively frees buffers in this level's metrics and all child levels. Buffers
// with standard capacity (BufferCap) are returned to the pool. Called by the
// retention worker to enforce retention policies.
//
// Parameters:
//   - t: Retention threshold timestamp (Unix seconds)
//
// Returns:
//   - int:   Total number of buffers freed in this subtree
//   - error: Non-nil on failure (propagated from children)
func (l *Level) free(t int64) (int, error) {
	n, _, err := l.freeAndCheckEmpty(t)
	return n, err
}

// freeAndCheckEmpty performs free() and atomically checks if the level is empty
// while still holding its own lock, avoiding a TOCTOU race between free() and
// a separate isEmpty() call.
func (l *Level) freeAndCheckEmpty(t int64) (int, bool, error) {
	l.lock.Lock()
	defer l.lock.Unlock()

	n := 0
	for i, b := range l.metrics {
		if b != nil {
			delme, m := b.free(t)
			n += m
			if delme {
				if cap(b.data) != BufferCap {
					b.data = make([]schema.Float, 0, BufferCap)
				}
				b.lastUsed = time.Now().Unix()
				bufferPool.Put(b)
				l.metrics[i] = nil
			}
		}
	}

	for key, child := range l.children {
		m, empty, err := child.freeAndCheckEmpty(t)
		n += m
		if err != nil {
			return n, false, err
		}
		if empty {
			delete(l.children, key)
		}
	}

	// Check emptiness while still holding the lock
	empty := len(l.children) == 0
	if empty {
		for _, b := range l.metrics {
			if b != nil {
				empty = false
				break
			}
		}
	}

	return n, empty, nil
}

// forceFree removes the oldest buffer from each metric chain in the subtree.
//
// Unlike free(), which removes based on time threshold, this unconditionally removes
// the oldest buffer in each chain. Used by MemoryUsageTracker when memory cap is
// exceeded and time-based retention is insufficient.
//
// Recursively processes current level's metrics and all child levels.
//
// Returns:
//   - int:   Total number of buffers freed in this subtree
//   - error: Non-nil on failure (propagated from children)
func (l *Level) forceFree() (int, error) {
	l.lock.Lock()
	defer l.lock.Unlock()

	n := 0

	// Iterate over metrics in the current level
	for i, b := range l.metrics {
		if b != nil {
			// Attempt to free the oldest buffer in this chain
			delme, freedCount := b.forceFreeOldest()
			n += freedCount

			// If delme is true, it means 'b' itself (the head) was the oldest
			// and needs to be removed from the slice.
			if delme {
				b.next = nil
				b.prev = nil
				if cap(b.data) != BufferCap {
					b.data = make([]schema.Float, 0, BufferCap)
				}
				b.lastUsed = time.Now().Unix()
				bufferPool.Put(b)
				l.metrics[i] = nil
			}
		}
	}

	// Recursively traverse children
	for _, child := range l.children {
		m, err := child.forceFree()
		n += m
		if err != nil {
			return n, err
		}
	}

	return n, nil
}

// sizeInBytes calculates the total memory usage of all buffers in the subtree.
//
// Recursively sums buffer data sizes (count of Float values × sizeof(Float)) across
// this level's metrics and all child levels. Used by MemoryUsageTracker to enforce
// memory cap limits.
//
// Returns:
//   - int64: Total bytes used by buffer data in this subtree
func (l *Level) sizeInBytes() int64 {
	l.lock.RLock()
	defer l.lock.RUnlock()
	size := int64(0)

	for _, b := range l.metrics {
		if b != nil {
			size += b.count() * int64(unsafe.Sizeof(schema.Float(0)))
			size += b.bufferCount() * int64(unsafe.Sizeof(buffer{}))
		}
	}

	for _, child := range l.children {
		size += child.sizeInBytes()
	}

	return size
}

// findLevel navigates to the level specified by selector, returning nil if not found.
//
// Read-only variant of findLevelOrCreate. Does not create missing levels.
// Recursively descends the tree following the selector path.
//
// Parameters:
//   - selector: Hierarchical path (e.g., []string{"emmy", "node001", "cpu0"})
//
// Returns:
//   - *Level: The target level, or nil if any component in the path does not exist
func (l *Level) findLevel(selector []string) *Level {
	if len(selector) == 0 {
		return l
	}

	l.lock.RLock()
	defer l.lock.RUnlock()

	lvl := l.children[selector[0]]
	if lvl == nil {
		return nil
	}

	return lvl.findLevel(selector[1:])
}

// findBuffers invokes callback on all buffers matching the selector pattern.
//
// Supports flexible selector patterns (from cc-lib/util.Selector):
//   - Exact match: Selector element with String set (e.g., "node001")
//   - Group match: Selector element with Group set (e.g., ["cpu0", "cpu2", "cpu4"])
//   - Wildcard:    Selector element with Any=true (matches all children)
//
// Empty selector (len==0) matches current level's buffer at 'offset' and recursively
// all descendant buffers at the same offset (used for aggregation queries).
//
// Parameters:
//   - selector: Pattern to match (consumed recursively)
//   - offset:   Metric index in metrics slice (from MetricConfig.offset)
//   - f:        Callback invoked on each matching buffer
//
// Returns:
//   - error: First error returned by callback, or nil if all succeeded
//
// Example:
//
//	// Find all cpu0 buffers across all hosts:
//	findBuffers([]Selector{{Any: true}, {String: "cpu0"}}, metricOffset, callback)
func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
	l.lock.RLock()
	defer l.lock.RUnlock()

	if len(selector) == 0 {
		b := l.metrics[offset]
		if b != nil {
			return f(b)
		}

		for _, lvl := range l.children {
			err := lvl.findBuffers(nil, offset, f)
			if err != nil {
				return err
			}
		}
		return nil
	}

	sel := selector[0]
	if len(sel.String) != 0 && l.children != nil {
		lvl, ok := l.children[sel.String]
		if ok {
			err := lvl.findBuffers(selector[1:], offset, f)
			if err != nil {
				return err
			}
		}
		return nil
	}

	if sel.Group != nil && l.children != nil {
		for _, key := range sel.Group {
			lvl, ok := l.children[key]
			if ok {
				err := lvl.findBuffers(selector[1:], offset, f)
				if err != nil {
					return err
				}
			}
		}
		return nil
	}

	if sel.Any && l.children != nil {
		for _, lvl := range l.children {
			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
				return err
			}
		}
		return nil
	}

	return nil
}