     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides an efficient in-memory time-series metric storage system
     7→// with support for hierarchical data organization, checkpointing, and archiving.
     8→//
     9→// The package organizes metrics in a tree structure (cluster → host → component) and
    10→// provides concurrent read/write access to metric data with configurable aggregation strategies.
    11→// Background goroutines handle periodic checkpointing (JSON or WAL/binary format), archiving old data,
    12→// and enforcing retention policies.
    13→//
    14→// Key features:
    15→//   - In-memory metric storage with configurable retention
    16→//   - Hierarchical data organization (selectors)
    17→//   - Concurrent checkpoint/archive workers
    18→//   - Support for sum and average aggregation
    19→//   - NATS integration for metric ingestion via InfluxDB line protocol
    20→package metricstore
    21→
    22→import (
    23→	"bytes"
    24→	"context"
    25→	"encoding/json"
    26→	"errors"
    27→	"fmt"
    28→	"runtime"
    29→	"runtime/debug"
    30→	"slices"
    31→	"sync"
    32→	"time"
    33→
    34→	"github.com/ClusterCockpit/cc-backend/internal/config"
    35→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    36→	"github.com/ClusterCockpit/cc-lib/v2/resampler"
    37→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    38→	"github.com/ClusterCockpit/cc-lib/v2/util"
    39→)
    40→
    41→// GlobalState holds the global state for the metric store with thread-safe access.
    42→type GlobalState struct {
    43→	mu                sync.RWMutex
    44→	lastRetentionTime int64
    45→	selectorsExcluded bool
    46→}
    47→
    48→var (
    49→	singleton  sync.Once
    50→	msInstance *MemoryStore
    51→	// shutdownFunc stores the context cancellation function created in Init
    52→	// and is called during Shutdown to cancel all background goroutines
    53→	shutdownFunc   context.CancelFunc
    54→	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
    55→	// Create a global instance
    56→	state = &GlobalState{}
    57→)
    58→
    59→// NodeProvider provides information about nodes currently in use by running jobs.
    60→//
    61→// This interface allows metricstore to query job information without directly
    62→// depending on the repository package, breaking the import cycle.
    63→//
    64→// Implementations should return nodes that are actively processing jobs started
    65→// before the given timestamp. These nodes will be excluded from retention-based
    66→// garbage collection to prevent data loss for jobs that are still running or
    67→// recently completed.
    68→type NodeProvider interface {
    69→	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
    70→	// that are currently in use by jobs that started before the given timestamp.
    71→	//
    72→	// Parameters:
    73→	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
    74→	//
    75→	// Returns:
    76→	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
    77→	//   - Error if the query fails
    78→	GetUsedNodes(ts int64) (map[string][]string, error)
    79→}
    80→
    81→// Metric represents a single metric data point to be written to the store.
    82→type Metric struct {
    83→	Name  string
    84→	Value schema.Float
    85→	// MetricConfig contains frequency and aggregation settings for this metric.
    86→	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
    87→	MetricConfig MetricConfig
    88→}
    89→
    90→// MemoryStore is the main in-memory time-series metric storage implementation.
    91→//
    92→// It organizes metrics in a hierarchical tree structure where each level represents
    93→// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
    94→// store multiple metrics as time-series buffers.
    95→//
    96→// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
    97→// All public methods are safe for concurrent use.
    98→type MemoryStore struct {
    99→	Metrics      map[string]MetricConfig
   100→	root         Level
   101→	nodeProvider NodeProvider
   102→}
   103→
   104→// Init initializes the metric store from configuration and starts background workers.
   105→//
   106→// This function must be called exactly once before any other metricstore operations.
   107→// It performs the following initialization steps:
   108→//  1. Validates and decodes the metric store configuration
   109→//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
   110→//  3. Loads metric configurations from all registered clusters
   111→//  4. Restores checkpoints within the retention window
   112→//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
   113→//  6. Optionally subscribes to NATS for real-time metric ingestion
   114→//
   115→// Parameters:
   116→//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig); may be nil to use defaults
   117→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
   118→//   - wg: WaitGroup that will be incremented for each background goroutine started
   119→//
   120→// The function will call cclog.Fatal on critical errors during initialization.
   121→// Use Shutdown() to cleanly stop all background workers started by Init().
   122→//
   123→// Note: Signal handling must be implemented by the caller. Call Shutdown() when
   124→// receiving termination signals to ensure checkpoint data is persisted.
   125→func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
   126→	startupTime := time.Now()
   127→
   128→	if rawConfig != nil {
   129→		config.Validate(configSchema, rawConfig)
   130→		dec := json.NewDecoder(bytes.NewReader(rawConfig))
   131→		dec.DisallowUnknownFields()
   132→		if err := dec.Decode(&Keys); err != nil {
   133→			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
   134→		}
   135→	}
   136→
   137→	// Set NumWorkers from config or use default
   138→	if Keys.NumWorkers <= 0 {
   139→		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
   140→	}
   141→	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
   142→
   143→	// Pass the config.MetricStoreKeys
   144→	InitMetrics(metrics)
   145→
   146→	ms := GetMemoryStore()
   147→
   148→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   149→	if err != nil {
   150→		cclog.Fatal(err)
   151→	}
   152→
   153→	restoreFrom := startupTime.Add(-d)
   154→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   155→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   156→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   157→	if err != nil {
   158→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   159→	} else {
   160→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   161→	}
   162→
   163→	// Try to use less memory by forcing a GC run here and then
   164→	// lowering the target percentage. The default of 100 means
   165→	// that only once the ratio of new allocations execeds the
   166→	// previously active heap, a GC is triggered.
   167→	// Forcing a GC here will set the "previously active heap"
   168→	// to a minumum.
   169→	// runtime.GC()
   170→
   171→	ctx, shutdown := context.WithCancel(context.Background())
   172→
   173→	Retention(wg, ctx)
   174→	Checkpointing(wg, ctx)
   175→	CleanUp(wg, ctx)
   176→	WALStaging(wg, ctx)
   177→	MemoryUsageTracker(wg, ctx)
   178→
   179→	// Note: Signal handling has been removed from this function.
   180→	// The caller is responsible for handling shutdown signals and calling
   181→	// the shutdown() function when appropriate.
   182→	// Store the shutdown function for later use by Shutdown()
   183→	shutdownFuncMu.Lock()
   184→	shutdownFunc = shutdown
   185→	shutdownFuncMu.Unlock()
   186→
   187→	if Keys.Subscriptions != nil {
   188→		wg.Go(func() {
   189→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   190→				cclog.Fatal(err)
   191→			}
   192→		})
   193→	}
   194→}
   195→
   196→// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
   197→//
   198→// This function must be called before GetMemoryStore() and can only be called once due to
   199→// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
   200→//
   201→// Parameters:
   202→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
   203→//
   204→// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
   205→//
   206→// After this call, the global msInstance is ready for use via GetMemoryStore().
   207→func InitMetrics(metrics map[string]MetricConfig) {
   208→	singleton.Do(func() {
   209→		offset := 0
   210→		for key, cfg := range metrics {
   211→			if cfg.Frequency == 0 {
   212→				panic("[METRICSTORE]> invalid frequency")
   213→			}
   214→
   215→			metrics[key] = MetricConfig{
   216→				Frequency:   cfg.Frequency,
   217→				Aggregation: cfg.Aggregation,
   218→				offset:      offset,
   219→			}
   220→			offset += 1
   221→		}
   222→
   223→		msInstance = &MemoryStore{
   224→			root: Level{
   225→				metrics:  make([]*buffer, len(metrics)),
   226→				children: make(map[string]*Level),
   227→			},
   228→			Metrics: metrics,
   229→		}
   230→	})
   231→}
   232→
   233→// GetMemoryStore returns the singleton MemoryStore instance.
   234→//
   235→// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
   236→//
   237→// This function is safe for concurrent use after initialization.
   238→func GetMemoryStore() *MemoryStore {
   239→	if msInstance == nil {
   240→		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
   241→	}
   242→
   243→	return msInstance
   244→}
   245→
   246→func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
   247→	if metric, ok := ms.Metrics[metricName]; ok {
   248→		return metric.Frequency, nil
   249→	}
   250→	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
   251→}
   252→
   253→// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
   254→// This must be called during initialization to provide job state information
   255→// for selective buffer retention during Free operations.
   256→// If not set, the Free function will fall back to freeing all buffers.
   257→func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
   258→	ms.nodeProvider = provider
   259→}
   260→
   261→// Shutdown performs a graceful shutdown of the metric store.
   262→//
   263→// This function cancels all background goroutines started by Init() and writes
   264→// a final checkpoint to disk before returning. It should be called when the
   265→// application receives a termination signal.
   266→//
   267→// The function will:
   268→//  1. Cancel the context to stop all background workers
   269→//  2. Close the WAL messages channel if using WAL format
   270→//  3. Write a final checkpoint to preserve in-memory data
   271→//  4. Log any errors encountered during shutdown
   272→//
   273→// Note: This function blocks until the final checkpoint is written.
   274→func Shutdown() {
   275→	totalStart := time.Now()
   276→
   277→	shutdownFuncMu.Lock()
   278→	defer shutdownFuncMu.Unlock()
   279→	if shutdownFunc != nil {
   280→		shutdownFunc()
   281→	}
   282→	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))
   283→
   284→	if Keys.Checkpoints.FileFormat == "wal" {
   285→		// Signal producers to stop sending before closing channels,
   286→		// preventing send-on-closed-channel panics from in-flight NATS workers.
   287→		walShuttingDown.Store(true)
   288→		// Brief grace period for in-flight DecodeLine calls to complete.
   289→		time.Sleep(100 * time.Millisecond)
   290→
   291→		for _, ch := range walShardChs {
   292→			close(ch)
   293→		}
   294→		drainStart := time.Now()
   295→		WaitForWALStagingDrain()
   296→		cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
   297→	}
   298→
   299→	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
   300→	checkpointStart := time.Now()
   301→	var files int
   302→	var err error
   303→
   304→	ms := GetMemoryStore()
   305→
   306→	lastCheckpointMu.Lock()
   307→	from := lastCheckpoint
   308→	lastCheckpointMu.Unlock()
   309→
   310→	if Keys.Checkpoints.FileFormat == "wal" {
   311→		// WAL files are deleted per-host inside ToCheckpointWAL workers.
   312→		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   313→	} else {
   314→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   315→	}
   316→
   317→	if err != nil {
   318→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
   319→	}
   320→	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
   321→}
   322→
   323→// Retention starts a background goroutine that periodically frees old metric data.
   324→//
   325→// This worker runs at half the retention interval and calls Free() to remove buffers
   326→// older than the configured retention time. It respects the NodeProvider to preserve
   327→// data for nodes with active jobs.
   328→//
   329→// Parameters:
   330→//   - wg: WaitGroup to signal completion when context is cancelled
   331→//   - ctx: Context for cancellation signal
   332→//
   333→// The goroutine exits when ctx is cancelled.
   334→func Retention(wg *sync.WaitGroup, ctx context.Context) {
   335→	ms := GetMemoryStore()
   336→
   337→	wg.Go(func() {
   338→		d, err := time.ParseDuration(Keys.RetentionInMemory)
   339→		if err != nil {
   340→			cclog.Fatal(err)
   341→		}
   342→		if d <= 0 {
   343→			return
   344→		}
   345→
   346→		tickInterval := d / 2
   347→		if tickInterval <= 0 {
   348→			return
   349→		}
   350→		ticker := time.NewTicker(tickInterval)
   351→		defer ticker.Stop()
   352→
   353→		for {
   354→			select {
   355→			case <-ctx.Done():
   356→				return
   357→			case <-ticker.C:
   358→				state.mu.Lock()
   359→
   360→				t := time.Now().Add(-d)
   361→
   362→				state.lastRetentionTime = t.Unix()
   363→
   364→				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
   365→
   366→				freed, err := Free(ms, t)
   367→				if err != nil {
   368→					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
   369→				} else {
   370→					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
   371→				}
   372→
   373→				state.mu.Unlock()
   374→
   375→				// Clean up the buffer pool
   376→				bufferPool.Clean(state.lastRetentionTime)
   377→			}
   378→		}
   379→	})
   380→}
   381→
   382→// MemoryUsageTracker starts a background goroutine that monitors memory usage.
   383→//
   384→// This worker checks actual process memory usage (via runtime.MemStats) periodically
   385→// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
   386→// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
   387→// performance issues.
   388→//
   389→// The tracker logs both actual memory usage (heap allocated) and metric data size for
   390→// visibility into memory overhead from Go runtime structures and allocations.
   391→//
   392→// Parameters:
   393→//   - wg: WaitGroup to signal completion when context is cancelled
   394→//   - ctx: Context for cancellation signal
   395→//
   396→// The goroutine exits when ctx is cancelled.
   397→func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
   398→	ms := GetMemoryStore()
   399→
   400→	wg.Go(func() {
   401→		normalInterval := DefaultMemoryUsageTrackerInterval
   402→		fastInterval := 30 * time.Second
   403→
   404→		if normalInterval <= 0 {
   405→			return
   406→		}
   407→
   408→		ticker := time.NewTicker(normalInterval)
   409→		defer ticker.Stop()
   410→		currentInterval := normalInterval
   411→
   412→		for {
   413→			select {
   414→			case <-ctx.Done():
   415→				return
   416→			case <-ticker.C:
   417→				var mem runtime.MemStats
   418→				runtime.ReadMemStats(&mem)
   419→				actualMemoryGB := float64(mem.Alloc) / 1e9
   420→				metricDataGB := ms.SizeInGB()
   421→				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
   422→
   423→				freedExcluded := 0
   424→				freedEmergency := 0
   425→				var err error
   426→
   427→				state.mu.RLock()
   428→				lastRetention := state.lastRetentionTime
   429→				selectorsExcluded := state.selectorsExcluded
   430→				state.mu.RUnlock()
   431→
   432→				if lastRetention != 0 && selectorsExcluded {
   433→					freedExcluded, err = ms.Free(nil, lastRetention)
   434→					if err != nil {
   435→						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
   436→					}
   437→
   438→					if freedExcluded > 0 {
   439→						debug.FreeOSMemory()
   440→						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
   441→					}
   442→				}
   443→
   444→				runtime.ReadMemStats(&mem)
   445→				actualMemoryGB = float64(mem.Alloc) / 1e9
   446→
   447→				if actualMemoryGB > float64(Keys.MemoryCap) {
   448→					cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)
   449→
   450→					// Use progressive time-based Free with increasing threshold
   451→					// instead of ForceFree loop — fewer tree traversals, more effective
   452→					d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
   453→					if parseErr != nil {
   454→						cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
   455→					} else {
   456→						thresholds := []float64{0.75, 0.5, 0.25}
   457→						for _, fraction := range thresholds {
   458→							threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
   459→							freed, freeErr := ms.Free(nil, threshold.Unix())
   460→							if freeErr != nil {
   461→								cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
   462→							}
   463→							freedEmergency += freed
   464→
   465→							bufferPool.Clear()
   466→							runtime.GC()
   467→							runtime.ReadMemStats(&mem)
   468→							actualMemoryGB = float64(mem.Alloc) / 1e9
   469→
   470→							if actualMemoryGB < float64(Keys.MemoryCap) {
   471→								break
   472→							}
   473→						}
   474→					}
   475→
   476→					bufferPool.Clear()
   477→					debug.FreeOSMemory()
   478→
   479→					runtime.ReadMemStats(&mem)
   480→					actualMemoryGB = float64(mem.Alloc) / 1e9
   481→
   482→					if actualMemoryGB >= float64(Keys.MemoryCap) {
   483→						cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
   484→					} else {
   485→						cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
   486→					}
   487→				}
   488→
   489→				// Adaptive ticker: check more frequently when memory is high
   490→				memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
   491→				if memoryRatio > 0.8 && currentInterval != fastInterval {
   492→					ticker.Reset(fastInterval)
   493→					currentInterval = fastInterval
   494→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
   495→				} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
   496→					ticker.Reset(normalInterval)
   497→					currentInterval = normalInterval
   498→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
   499→				}
   500→			}
   501→		}
   502→	})
   503→}
   504→
   505→// Free removes metric data older than the given time while preserving data for active nodes.
   506→//
   507→// This function implements intelligent retention by consulting the NodeProvider (if configured)
   508→// to determine which nodes are currently in use by running jobs. Data for these nodes is
   509→// preserved even if older than the retention time.
   510→//
   511→// Parameters:
   512→//   - ms: The MemoryStore instance
   513→//   - t: Time threshold - buffers with data older than this will be freed
   514→//
   515→// Returns:
   516→//   - Number of buffers freed
   517→//   - Error if NodeProvider query fails
   518→//
   519→// Behavior:
   520→//   - If no NodeProvider is set: frees all buffers older than t
   521→//   - If NodeProvider returns empty map: frees all buffers older than t
   522→//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
   523→func Free(ms *MemoryStore, t time.Time) (int, error) {
   524→	// If no NodeProvider is configured, free all buffers older than t
   525→	if ms.nodeProvider == nil {
   526→		return ms.Free(nil, t.Unix())
   527→	}
   528→
   529→	excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
   530→	if err != nil {
   531→		return 0, err
   532→	}
   533→
   534→	switch lenMap := len(excludeSelectors); lenMap {
   535→
   536→	// If the length of the map returned by GetUsedNodes() is 0,
   537→	// then use default Free method with nil selector
   538→	case 0:
   539→		state.selectorsExcluded = false
   540→		return ms.Free(nil, t.Unix())
   541→
   542→	// Else formulate selectors, exclude those from the map
   543→	// and free the rest of the selectors
   544→	default:
   545→		state.selectorsExcluded = true
   546→		selectors := GetSelectors(ms, excludeSelectors)
   547→		return FreeSelected(ms, selectors, t)
   548→	}
   549→}
   550→
   551→// FreeSelected frees buffers for specific selectors while preserving others.
   552→//
   553→// This function is used when we want to retain some specific nodes beyond the retention time.
   554→// It iterates through the provided selectors and frees their associated buffers.
   555→//
   556→// Parameters:
   557→//   - ms: The MemoryStore instance
   558→//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
   559→//   - t: Time threshold for freeing buffers
   560→//
   561→// Returns the total number of buffers freed and any error encountered.
   562→func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
   563→	freed := 0
   564→
   565→	for _, selector := range selectors {
   566→
   567→		freedBuffers, err := ms.Free(selector, t.Unix())
   568→		if err != nil {
   569→			cclog.Errorf("error while freeing selected buffers: %#v", err)
   570→		}
   571→		freed += freedBuffers
   572→
   573→	}
   574→
   575→	return freed, nil
   576→}
   577→
   578→// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
   579→//
   580→// This function generates a list of selectors whose buffers should be freed by excluding
   581→// selectors that correspond to nodes currently in use by running jobs.
   582→//
   583→// Parameters:
   584→//   - ms: The MemoryStore instance
   585→//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
   586→//
   587→// Returns a list of selectors ([]string paths) that can be safely freed.
   588→//
   589→// Example:
   590→//
   591→//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
   592→//	and excludeSelectors contains {"emmy": ["node001"]},
   593→//	then only [["emmy", "node002"]] is returned.
   594→func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
   595→	allSelectors := ms.GetPaths(2)
   596→
   597→	filteredSelectors := make([][]string, 0, len(allSelectors))
   598→
   599→	for _, path := range allSelectors {
   600→		if len(path) < 2 {
   601→			continue
   602→		}
   603→
   604→		key := path[0]   // The "Key" (Level 1)
   605→		value := path[1] // The "Value" (Level 2)
   606→
   607→		exclude := false
   608→
   609→		// Check if the key exists in our exclusion map
   610→		if excludedValues, exists := excludeSelectors[key]; exists {
   611→			// The key exists, now check if the specific value is in the exclusion list
   612→			if slices.Contains(excludedValues, value) {
   613→				exclude = true
   614→			}
   615→		}
   616→
   617→		if !exclude {
   618→			filteredSelectors = append(filteredSelectors, path)
   619→		}
   620→	}
   621→
   622→	return filteredSelectors
   623→}
   624→
   625→// GetPaths returns a list of lists (paths) to the specified depth.
   626→func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
   627→	var results [][]string
   628→
   629→	// Start recursion. Initial path is empty.
   630→	// We treat Root as depth 0.
   631→	ms.root.collectPaths(0, targetDepth, []string{}, &results)
   632→
   633→	return results
   634→}
   635→
   636→// Write all values in `metrics` to the level specified by `selector` for time `ts`.
   637→// Look at `findLevelOrCreate` for how selectors work.
   638→func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
   639→	var ok bool
   640→	for i, metric := range metrics {
   641→		if metric.MetricConfig.Frequency == 0 {
   642→			metric.MetricConfig, ok = m.Metrics[metric.Name]
   643→			if !ok {
   644→				cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
   645→				metric.MetricConfig.Frequency = 0
   646→			}
   647→			metrics[i] = metric
   648→		}
   649→	}
   650→
   651→	return m.WriteToLevel(&m.root, selector, ts, metrics)
   652→}
   653→
   654→func (m *MemoryStore) GetLevel(selector []string) *Level {
   655→	return m.root.findLevelOrCreate(selector, len(m.Metrics))
   656→}
   657→
   658→// WriteToLevel assumes that `minfo` in `metrics` is filled in
   659→func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
   660→	l = l.findLevelOrCreate(selector, len(m.Metrics))
   661→	l.lock.Lock()
   662→	defer l.lock.Unlock()
   663→
   664→	for _, metric := range metrics {
   665→		if metric.MetricConfig.Frequency == 0 {
   666→			continue
   667→		}
   668→
   669→		b := l.metrics[metric.MetricConfig.offset]
   670→		if b == nil {
   671→			// First write to this metric and level
   672→			b = newBuffer(ts, metric.MetricConfig.Frequency)
   673→			l.metrics[metric.MetricConfig.offset] = b
   674→		}
   675→
   676→		nb, err := b.write(ts, metric.Value)
   677→		if err != nil {
   678→			return err
   679→		}
   680→
   681→		// Last write created a new buffer...
   682→		if b != nb {
   683→			l.metrics[metric.MetricConfig.offset] = nb
   684→		}
   685→	}
   686→	return nil
   687→}
   688→
   689→// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
   690→// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
   691→// The second and third return value are the actual from/to for the data. Those can be different from
   692→// the range asked for if no data was available.
   693→func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
   694→	if from > to {
   695→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
   696→	}
   697→
   698→	minfo, ok := m.Metrics[metric]
   699→	if !ok {
   700→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
   701→	}
   702→
   703→	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
   704→
   705→	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
   706→		cdata, cfrom, cto, err := b.read(from, to, data)
   707→		if err != nil {
   708→			return err
   709→		}
   710→
   711→		if n == 0 {
   712→			from, to = cfrom, cto
   713→		} else if from != cfrom || to != cto || len(data) != len(cdata) {
   714→			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
   715→			if missingfront != 0 {
   716→				return ErrDataDoesNotAlignMissingFront
   717→			}
   718→
   719→			newlen := len(cdata) - missingback
   720→			if newlen < 1 {
   721→				return ErrDataDoesNotAlignMissingBack
   722→			}
   723→			cdata = cdata[0:newlen]
   724→			if len(cdata) != len(data) {
   725→				return ErrDataDoesNotAlignDataLenMismatch
   726→			}
   727→
   728→			from, to = cfrom, cto
   729→		}
   730→
   731→		data = cdata
   732→		n += 1
   733→		return nil
   734→	})
   735→
   736→	if err != nil {
   737→		return nil, 0, 0, 0, err
   738→	} else if n == 0 {
   739→		return nil, 0, 0, 0, ErrNoHostOrMetric
   740→	} else if n > 1 {
   741→		if minfo.Aggregation == AvgAggregation {
   742→			normalize := 1. / schema.Float(n)
   743→			for i := 0; i < len(data); i++ {
   744→				data[i] *= normalize
   745→			}
   746→		} else if minfo.Aggregation != SumAggregation {
   747→			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
   748→		}
   749→	}
   750→
   751→	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
   752→	if err != nil {
   753→		return nil, 0, 0, 0, err
   754→	}
   755→
   756→	return data, from, to, resolution, nil
   757→}
   758→
   759→// Free releases all buffers for the selected level and all its children that
   760→// contain only values older than `t`.
   761→func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
   762→	return m.GetLevel(selector).free(t)
   763→}
   764→
   765→// ForceFree unconditionally removes the oldest buffer from each metric chain.
   766→func (m *MemoryStore) ForceFree() (int, error) {
   767→	return m.GetLevel(nil).forceFree()
   768→}
   769→
   770→func (m *MemoryStore) FreeAll() error {
   771→	m.root.lock.Lock()
   772→	defer m.root.lock.Unlock()
   773→
   774→	for k := range m.root.children {
   775→		delete(m.root.children, k)
   776→	}
   777→
   778→	return nil
   779→}
   780→
   781→func (m *MemoryStore) SizeInBytes() int64 {
   782→	return m.root.sizeInBytes()
   783→}
   784→
   785→func (m *MemoryStore) SizeInGB() float64 {
   786→	return float64(m.root.sizeInBytes()) / 1e9
   787→}
   788→
   789→// ListChildren , given a selector, returns a list of all children of the level
   790→// selected.
   791→func (m *MemoryStore) ListChildren(selector []string) []string {
   792→	lvl := &m.root
   793→	for lvl != nil && len(selector) != 0 {
   794→		lvl.lock.RLock()
   795→		next := lvl.children[selector[0]]
   796→		lvl.lock.RUnlock()
   797→		lvl = next
   798→		selector = selector[1:]
   799→	}
   800→
   801→	if lvl == nil {
   802→		return nil
   803→	}
   804→
   805→	lvl.lock.RLock()
   806→	defer lvl.lock.RUnlock()
   807→
   808→	children := make([]string, 0, len(lvl.children))
   809→	for child := range lvl.children {
   810→		children = append(children, child)
   811→	}
   812→
   813→	return children
   814→}
   815→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements checkpoint persistence for the in-memory metric store.
     7→//
     8→// Checkpoints enable graceful restarts by periodically saving in-memory metric
     9→// data to disk in JSON or binary format. The checkpoint system:
    10→//
    11→// Key Features:
    12→//   - Periodic background checkpointing via the Checkpointing() worker
    13→//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
    14→//   - Parallel checkpoint creation and loading using worker pools
    15→//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
    16→//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
    17→//   - Only saves unarchived data (archived data is already persisted elsewhere)
    18→//   - GC optimization during loading to prevent excessive heap growth
    19→//
    20→// Checkpoint Workflow:
    21→//  1. Init() loads checkpoints within retention window at startup
    22→//  2. Checkpointing() worker periodically saves new data
    23→//  3. Shutdown() writes final checkpoint before exit
    24→//
    25→// File Organization:
    26→//
    27→//	checkpoints/
    28→//	  cluster1/
    29→//	    host001/
    30→//	      1234567890.json  (JSON format: full subtree snapshot)
    31→//	      1234567890.bin   (binary format: full subtree snapshot)
    32→//	      current.wal      (WAL format: append-only per-entry log)
    33→//	    host002/
    34→//	      ...
    35→package metricstore
    36→
    37→import (
    38→	"bufio"
    39→	"context"
    40→	"encoding/json"
    41→	"errors"
    42→	"fmt"
    43→	"io/fs"
    44→	"os"
    45→	"path"
    46→	"path/filepath"
    47→	"sort"
    48→	"strconv"
    49→	"strings"
    50→	"sync"
    51→	"sync/atomic"
    52→	"time"
    53→
    54→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    55→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    56→)
    57→
    58→const (
    59→	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
    60→	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
    61→	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
    62→)
    63→
    64→// CheckpointMetrics represents metric data in a checkpoint file.
    65→// Whenever the structure changes, update MarshalJSON as well!
    66→type CheckpointMetrics struct {
    67→	Data      []schema.Float `json:"data"`
    68→	Frequency int64          `json:"frequency"`
    69→	Start     int64          `json:"start"`
    70→}
    71→
    72→// CheckpointFile represents the hierarchical structure of a checkpoint file.
    73→// It mirrors the Level tree structure from the MemoryStore.
    74→type CheckpointFile struct {
    75→	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
    76→	Children map[string]*CheckpointFile    `json:"children"`
    77→	From     int64                         `json:"from"`
    78→	To       int64                         `json:"to"`
    79→}
    80→
    81→// lastCheckpoint tracks the timestamp of the last checkpoint creation.
    82→var (
    83→	lastCheckpoint   time.Time
    84→	lastCheckpointMu sync.Mutex
    85→)
    86→
    87→// Checkpointing starts a background worker that periodically saves metric data to disk.
    88→//
    89→// The checkpoint interval is read from Keys.CheckpointInterval (default: 12 hours).
    90→//
    91→// Format behaviour:
    92→//   - "json": Writes a JSON snapshot file per host every interval
    93→//   - "wal":  Writes a binary snapshot file per host every interval, then rotates
    94→//     the current.wal files for all successfully checkpointed hosts
    95→func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
    96→	lastCheckpointMu.Lock()
    97→	lastCheckpoint = time.Now()
    98→	lastCheckpointMu.Unlock()
    99→
   100→	ms := GetMemoryStore()
   101→
   102→	wg.Go(func() {
   103→		d := 12 * time.Hour // default checkpoint interval
   104→		if Keys.CheckpointInterval != "" {
   105→			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
   106→			if err != nil {
   107→				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
   108→			} else {
   109→				d = parsed
   110→			}
   111→		}
   112→
   113→		ticker := time.NewTicker(d)
   114→		defer ticker.Stop()
   115→
   116→		for {
   117→			select {
   118→			case <-ctx.Done():
   119→				return
   120→			case <-ticker.C:
   121→				lastCheckpointMu.Lock()
   122→				from := lastCheckpoint
   123→				lastCheckpointMu.Unlock()
   124→
   125→				now := time.Now()
   126→				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
   127→
   128→				if Keys.Checkpoints.FileFormat == "wal" {
   129→					// Pause WAL writes: the binary snapshot captures all in-memory
   130→					// data, so WAL records written during checkpoint are redundant
   131→					// and would be deleted during rotation anyway.
   132→					walCheckpointActive.Store(true)
   133→					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   134→					if err != nil {
   135→						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
   136→					}
   137→					if n > 0 {
   138→						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
   139→						lastCheckpointMu.Lock()
   140→						lastCheckpoint = now
   141→						lastCheckpointMu.Unlock()
   142→						// Rotate WAL files for successfully checkpointed hosts.
   143→						RotateWALFiles(hostDirs)
   144→					}
   145→					walCheckpointActive.Store(false)
   146→					walDropped.Store(0)
   147→				} else {
   148→					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   149→					if err != nil {
   150→						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
   151→					} else {
   152→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
   153→						lastCheckpointMu.Lock()
   154→						lastCheckpoint = now
   155→						lastCheckpointMu.Unlock()
   156→					}
   157→				}
   158→			}
   159→		}
   160→	})
   161→}
   162→
   163→// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
   164→//
   165→// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
   166→// This method manually constructs JSON to avoid allocations and interface conversions.
   167→func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
   168→	buf := make([]byte, 0, 128+len(cm.Data)*8)
   169→	buf = append(buf, `{"frequency":`...)
   170→	buf = strconv.AppendInt(buf, cm.Frequency, 10)
   171→	buf = append(buf, `,"start":`...)
   172→	buf = strconv.AppendInt(buf, cm.Start, 10)
   173→	buf = append(buf, `,"data":[`...)
   174→	for i, x := range cm.Data {
   175→		if i != 0 {
   176→			buf = append(buf, ',')
   177→		}
   178→		if x.IsNaN() {
   179→			buf = append(buf, `null`...)
   180→		} else {
   181→			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
   182→		}
   183→	}
   184→	buf = append(buf, `]}`...)
   185→	return buf, nil
   186→}
   187→
   188→// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format).
   189→//
   190→// Metrics at root and cluster levels are skipped. One file per host is created.
   191→// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
   192→// at a time, allowing concurrent writes/reads to other hosts.
   193→//
   194→// Returns the number of checkpoint files created and any errors encountered.
   195→func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
   196→	// Pre-calculate capacity by counting cluster/host pairs
   197→	m.root.lock.RLock()
   198→	totalHosts := 0
   199→	for _, l1 := range m.root.children {
   200→		l1.lock.RLock()
   201→		totalHosts += len(l1.children)
   202→		l1.lock.RUnlock()
   203→	}
   204→	m.root.lock.RUnlock()
   205→
   206→	levels := make([]*Level, 0, totalHosts)
   207→	selectors := make([][]string, 0, totalHosts)
   208→
   209→	m.root.lock.RLock()
   210→	for sel1, l1 := range m.root.children {
   211→		l1.lock.RLock()
   212→		for sel2, l2 := range l1.children {
   213→			levels = append(levels, l2)
   214→			selectors = append(selectors, []string{sel1, sel2})
   215→		}
   216→		l1.lock.RUnlock()
   217→	}
   218→	m.root.lock.RUnlock()
   219→
   220→	type workItem struct {
   221→		level    *Level
   222→		dir      string
   223→		selector []string
   224→	}
   225→
   226→	n, errs := int32(0), int32(0)
   227→
   228→	var wg sync.WaitGroup
   229→	wg.Add(Keys.NumWorkers)
   230→	work := make(chan workItem, Keys.NumWorkers*2)
   231→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   232→		go func() {
   233→			defer wg.Done()
   234→
   235→			for workItem := range work {
   236→				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
   237→					if err == ErrNoNewArchiveData {
   238→						continue
   239→					}
   240→
   241→					cclog.Errorf("[METRICSTORE]> error while checkpointing %#v: %s", workItem.selector, err.Error())
   242→					atomic.AddInt32(&errs, 1)
   243→				} else {
   244→					atomic.AddInt32(&n, 1)
   245→				}
   246→			}
   247→		}()
   248→	}
   249→
   250→	for i := 0; i < len(levels); i++ {
   251→		dir := path.Join(dir, path.Join(selectors[i]...))
   252→		work <- workItem{
   253→			level:    levels[i],
   254→			dir:      dir,
   255→			selector: selectors[i],
   256→		}
   257→	}
   258→
   259→	close(work)
   260→	wg.Wait()
   261→
   262→	if errs > 0 {
   263→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   264→	}
   265→	return int(n), nil
   266→}
   267→
   268→// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
   269→// Skips metrics that are already archived. Returns nil if no unarchived data exists.
   270→func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
   271→	l.lock.RLock()
   272→	defer l.lock.RUnlock()
   273→
   274→	retval := &CheckpointFile{
   275→		From:     from,
   276→		To:       to,
   277→		Metrics:  make(map[string]*CheckpointMetrics),
   278→		Children: make(map[string]*CheckpointFile),
   279→	}
   280→
   281→	for metric, minfo := range m.Metrics {
   282→		b := l.metrics[minfo.offset]
   283→		if b == nil {
   284→			continue
   285→		}
   286→
   287→		allArchived := true
   288→		b.iterFromTo(from, to, func(b *buffer) error {
   289→			if !b.archived {
   290→				allArchived = false
   291→				return fmt.Errorf("stop") // Early termination signal
   292→			}
   293→			return nil
   294→		})
   295→
   296→		if allArchived {
   297→			continue
   298→		}
   299→
   300→		data := make([]schema.Float, (to-from)/b.frequency+1)
   301→		data, start, end, err := b.read(from, to, data)
   302→		if err != nil {
   303→			return nil, err
   304→		}
   305→
   306→		for i := int((end - start) / b.frequency); i < len(data); i++ {
   307→			data[i] = schema.NaN
   308→		}
   309→
   310→		retval.Metrics[metric] = &CheckpointMetrics{
   311→			Frequency: b.frequency,
   312→			Start:     start,
   313→			Data:      data,
   314→		}
   315→	}
   316→
   317→	for name, child := range l.children {
   318→		val, err := child.toCheckpointFile(from, to, m)
   319→		if err != nil {
   320→			return nil, err
   321→		}
   322→
   323→		if val != nil {
   324→			retval.Children[name] = val
   325→		}
   326→	}
   327→
   328→	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
   329→		return nil, nil
   330→	}
   331→
   332→	return retval, nil
   333→}
   334→
   335→// toCheckpoint writes a Level's data to a JSON checkpoint file.
   336→// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.
   337→func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
   338→	cf, err := l.toCheckpointFile(from, to, m)
   339→	if err != nil {
   340→		return err
   341→	}
   342→
   343→	if cf == nil {
   344→		return ErrNoNewArchiveData
   345→	}
   346→
   347→	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
   348→	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   349→	if err != nil && os.IsNotExist(err) {
   350→		err = os.MkdirAll(dir, CheckpointDirPerms)
   351→		if err == nil {
   352→			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   353→		}
   354→	}
   355→	if err != nil {
   356→		return err
   357→	}
   358→	defer f.Close()
   359→
   360→	bw := bufio.NewWriter(f)
   361→	if err = json.NewEncoder(bw).Encode(cf); err != nil {
   362→		return err
   363→	}
   364→
   365→	return bw.Flush()
   366→}
   367→
   368→// enqueueCheckpointHosts traverses checkpoint directory and enqueues cluster/host pairs.
   369→// Returns error if directory structure is invalid.
   370→func enqueueCheckpointHosts(dir string, work chan<- [2]string) error {
   371→	clustersDir, err := os.ReadDir(dir)
   372→	if err != nil {
   373→		return err
   374→	}
   375→
   376→	for _, clusterDir := range clustersDir {
   377→		if !clusterDir.IsDir() {
   378→			return errors.New("[METRICSTORE]> expected only directories at first level of checkpoints/ directory")
   379→		}
   380→
   381→		hostsDir, err := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
   382→		if err != nil {
   383→			return err
   384→		}
   385→
   386→		for _, hostDir := range hostsDir {
   387→			if !hostDir.IsDir() {
   388→				return errors.New("[METRICSTORE]> expected only directories at second level of checkpoints/ directory")
   389→			}
   390→
   391→			work <- [2]string{clusterDir.Name(), hostDir.Name()}
   392→		}
   393→	}
   394→
   395→	return nil
   396→}
   397→
   398→// FromCheckpoint loads checkpoint files from disk into memory in parallel.
   399→//
   400→// Uses worker pool to load cluster/host combinations. Returns number of files
   401→// loaded and any errors.
   402→func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
   403→	var wg sync.WaitGroup
   404→	work := make(chan [2]string, Keys.NumWorkers*4)
   405→	n, errs := int32(0), int32(0)
   406→
   407→	wg.Add(Keys.NumWorkers)
   408→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   409→		go func() {
   410→			defer wg.Done()
   411→			for host := range work {
   412→				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
   413→				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
   414→				if err != nil {
   415→					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
   416→					atomic.AddInt32(&errs, 1)
   417→				}
   418→				atomic.AddInt32(&n, int32(nn))
   419→			}
   420→		}()
   421→	}
   422→
   423→	err := enqueueCheckpointHosts(dir, work)
   424→	close(work)
   425→	wg.Wait()
   426→
   427→	if err != nil {
   428→		return int(n), err
   429→	}
   430→
   431→	if errs > 0 {
   432→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   433→	}
   434→	return int(n), nil
   435→}
   436→
   437→// FromCheckpointFiles is the main entry point for loading checkpoints at startup.
   438→//
   439→// Creates checkpoint directory if it doesn't exist. This function must be called
   440→// before any writes or reads, and can only be called once.
   441→func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
   442→	if _, err := os.Stat(dir); os.IsNotExist(err) {
   443→		err := os.MkdirAll(dir, CheckpointDirPerms)
   444→		if err != nil {
   445→			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
   446→		}
   447→		cclog.Debugf("[METRICSTORE]> %#v Directory created successfully", dir)
   448→	}
   449→
   450→	return m.FromCheckpoint(dir, from)
   451→}
   452→
   453→func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
   454→	br := bufio.NewReader(f)
   455→	cf := &CheckpointFile{}
   456→	if err := json.NewDecoder(br).Decode(cf); err != nil {
   457→		return err
   458→	}
   459→
   460→	if cf.To != 0 && cf.To < from {
   461→		return nil
   462→	}
   463→
   464→	if err := l.loadFile(cf, m); err != nil {
   465→		return err
   466→	}
   467→
   468→	return nil
   469→}
   470→
   471→func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
   472→	for name, metric := range cf.Metrics {
   473→		n := len(metric.Data)
   474→		b := &buffer{
   475→			frequency: metric.Frequency,
   476→			start:     metric.Start,
   477→			data:      metric.Data[0:n:n],
   478→			prev:      nil,
   479→			next:      nil,
   480→			archived:  true,
   481→		}
   482→
   483→		minfo, ok := m.Metrics[name]
   484→		if !ok {
   485→			continue
   486→		}
   487→
   488→		prev := l.metrics[minfo.offset]
   489→		if prev == nil {
   490→			l.metrics[minfo.offset] = b
   491→		} else {
   492→			if prev.start > b.start {
   493→				return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start)
   494→			}
   495→
   496→			b.prev = prev
   497→			prev.next = b
   498→		}
   499→		l.metrics[minfo.offset] = b
   500→	}
   501→
   502→	if len(cf.Children) > 0 && l.children == nil {
   503→		l.children = make(map[string]*Level)
   504→	}
   505→
   506→	for sel, childCf := range cf.Children {
   507→		child, ok := l.children[sel]
   508→		if !ok {
   509→			child = &Level{
   510→				metrics:  make([]*buffer, len(m.Metrics)),
   511→				children: nil,
   512→			}
   513→			l.children[sel] = child
   514→		}
   515→
   516→		if err := child.loadFile(childCf, m); err != nil {
   517→			return err
   518→		}
   519→	}
   520→
   521→	return nil
   522→}
   523→
   524→// fromCheckpoint loads all checkpoint files (JSON, binary snapshot, WAL) for a
   525→// single host directory. Snapshot files are loaded first (sorted by timestamp),
   526→// then current.wal is replayed on top.
   527→func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {
   528→	direntries, err := os.ReadDir(dir)
   529→	if err != nil {
   530→		if os.IsNotExist(err) {
   531→			return 0, nil
   532→		}
   533→		return 0, err
   534→	}
   535→
   536→	allFiles := make([]fs.DirEntry, 0)
   537→	var walEntry fs.DirEntry
   538→	filesLoaded := 0
   539→
   540→	for _, e := range direntries {
   541→		if e.IsDir() {
   542→			// Legacy: skip subdirectories (only used by old Avro format).
   543→			// These are ignored; their data is not loaded.
   544→			cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir)
   545→			continue
   546→		}
   547→
   548→		name := e.Name()
   549→		if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") {
   550→			allFiles = append(allFiles, e)
   551→		} else if name == "current.wal" {
   552→			walEntry = e
   553→		}
   554→		// Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes).
   555→	}
   556→
   557→	files, err := findFiles(allFiles, from, true)
   558→	if err != nil {
   559→		return filesLoaded, err
   560→	}
   561→
   562→	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
   563→		".json": l.loadJSONFile,
   564→		".bin":  l.loadBinaryFile,
   565→	}
   566→
   567→	for _, filename := range files {
   568→		ext := filepath.Ext(filename)
   569→		loader := loaders[ext]
   570→		if loader == nil {
   571→			cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename)
   572→			continue
   573→		}
   574→
   575→		err := func() error {
   576→			f, err := os.Open(path.Join(dir, filename))
   577→			if err != nil {
   578→				return err
   579→			}
   580→			defer f.Close()
   581→			return loader(m, f, from)
   582→		}()
   583→		if err != nil {
   584→			return filesLoaded, err
   585→		}
   586→		filesLoaded++
   587→	}
   588→
   589→	// Replay WAL after all snapshot files so it fills in data since the last snapshot.
   590→	if walEntry != nil {
   591→		err := func() error {
   592→			f, err := os.Open(path.Join(dir, walEntry.Name()))
   593→			if err != nil {
   594→				return err
   595→			}
   596→			defer f.Close()
   597→			return l.loadWALFile(m, f, from)
   598→		}()
   599→		if err != nil {
   600→			// WAL errors are non-fatal: the snapshot already loaded the bulk of data.
   601→			cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err)
   602→		} else {
   603→			filesLoaded++
   604→		}
   605→	}
   606→
   607→	return filesLoaded, nil
   608→}
   609→
   610→// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename.
   611→// Supports ".json" (format: "<ts>.json") and ".bin" (format: "<ts>.bin").
   612→func parseTimestampFromFilename(name string) (int64, error) {
   613→	switch {
   614→	case strings.HasSuffix(name, ".json"):
   615→		return strconv.ParseInt(name[:len(name)-5], 10, 64)
   616→	case strings.HasSuffix(name, ".bin"):
   617→		return strconv.ParseInt(name[:len(name)-4], 10, 64)
   618→	default:
   619→		return 0, fmt.Errorf("unknown checkpoint extension for file %q", name)
   620→	}
   621→}
   622→
   623→// findFiles returns filenames from direntries whose timestamps satisfy the filter.
   624→// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the
   625→// last file before t if t falls between two files).
   626→func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {
   627→	nums := map[string]int64{}
   628→	for _, e := range direntries {
   629→		name := e.Name()
   630→		if !strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".bin") {
   631→			continue
   632→		}
   633→
   634→		ts, err := parseTimestampFromFilename(name)
   635→		if err != nil {
   636→			return nil, err
   637→		}
   638→		nums[name] = ts
   639→	}
   640→
   641→	sort.Slice(direntries, func(i, j int) bool {
   642→		a, b := direntries[i], direntries[j]
   643→		return nums[a.Name()] < nums[b.Name()]
   644→	})
   645→
   646→	if len(nums) == 0 {
   647→		return nil, nil
   648→	}
   649→
   650→	filenames := make([]string, 0)
   651→
   652→	for i, e := range direntries {
   653→		ts1 := nums[e.Name()]
   654→
   655→		if findMoreRecentFiles && t <= ts1 {
   656→			filenames = append(filenames, e.Name())
   657→		} else if !findMoreRecentFiles && ts1 <= t && ts1 != 0 {
   658→			filenames = append(filenames, e.Name())
   659→		}
   660→
   661→		if i == len(direntries)-1 {
   662→			continue
   663→		}
   664→
   665→		enext := direntries[i+1]
   666→		ts2 := nums[enext.Name()]
   667→
   668→		if findMoreRecentFiles {
   669→			if ts1 < t && t < ts2 {
   670→				filenames = append(filenames, e.Name())
   671→			}
   672→		}
   673→	}
   674→
   675→	return filenames, nil
   676→}
   677→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


Error: File content (10360 tokens) exceeds maximum allowed tokens (10000). Use offset and limit parameters to read specific portions of the file, or search for specific content instead of reading the whole file.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements the cleanup (archiving or deletion) of old checkpoint files.
     7→//
     8→// The CleanUp worker runs on a timer equal to RetentionInMemory. In "archive" mode
     9→// it converts checkpoint files older than the retention window into per-cluster
    10→// Parquet files and then deletes the originals. In "delete" mode it simply removes
    11→// old checkpoint files.
    12→package metricstore
    13→
    14→import (
    15→	"context"
    16→	"errors"
    17→	"fmt"
    18→	"os"
    19→	"path/filepath"
    20→	"strings"
    21→	"sync"
    22→	"sync/atomic"
    23→	"time"
    24→
    25→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    26→)
    27→
    28→// CleanUp starts a background worker that periodically removes or archives
    29→// checkpoint files older than the configured retention window.
    30→//
    31→// In "archive" mode, old checkpoint files are converted to Parquet and stored
    32→// under Keys.Cleanup.RootDir. In "delete" mode they are simply removed.
    33→// The cleanup interval equals Keys.RetentionInMemory.
    34→func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
    35→	if Keys.Cleanup.Mode == "archive" {
    36→		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
    37→		// Run as Archiver
    38→		cleanUpWorker(wg, ctx,
    39→			Keys.RetentionInMemory,
    40→			"archiving",
    41→			Keys.Cleanup.RootDir,
    42→			false,
    43→		)
    44→	} else {
    45→		// Run as Deleter
    46→		cleanUpWorker(wg, ctx,
    47→			Keys.RetentionInMemory,
    48→			"deleting",
    49→			"",
    50→			true,
    51→		)
    52→	}
    53→}
    54→
    55→// cleanUpWorker takes simple values to configure what it does
    56→func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
    57→	wg.Go(func() {
    58→		d, err := time.ParseDuration(interval)
    59→		if err != nil {
    60→			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
    61→		}
    62→		if d <= 0 {
    63→			return
    64→		}
    65→
    66→		ticker := time.NewTicker(d)
    67→		defer ticker.Stop()
    68→
    69→		for {
    70→			select {
    71→			case <-ctx.Done():
    72→				return
    73→			case <-ticker.C:
    74→				t := time.Now().Add(-d)
    75→				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))
    76→
    77→				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)
    78→
    79→				if err != nil {
    80→					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
    81→				} else {
    82→					if delete {
    83→						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
    84→					} else {
    85→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
    86→					}
    87→				}
    88→			}
    89→		}
    90→	})
    91→}
    92→
    93→var ErrNoNewArchiveData error = errors.New("all data already archived")
    94→
    95→// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
    96→// When archiving, consolidates all hosts per cluster into a single Parquet file.
    97→func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
    98→	if deleteInstead {
    99→		return deleteCheckpoints(checkpointsDir, from)
   100→	}
   101→
   102→	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
   103→}
   104→
   105→// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
   106→func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
   107→	entries1, err := os.ReadDir(checkpointsDir)
   108→	if err != nil {
   109→		return 0, err
   110→	}
   111→
   112→	type workItem struct {
   113→		dir           string
   114→		cluster, host string
   115→	}
   116→
   117→	var wg sync.WaitGroup
   118→	n, errs := int32(0), int32(0)
   119→	work := make(chan workItem, Keys.NumWorkers)
   120→
   121→	wg.Add(Keys.NumWorkers)
   122→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   123→		go func() {
   124→			defer wg.Done()
   125→			for item := range work {
   126→				entries, err := os.ReadDir(item.dir)
   127→				if err != nil {
   128→					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
   129→					atomic.AddInt32(&errs, 1)
   130→					continue
   131→				}
   132→
   133→				files, err := findFiles(entries, from, false)
   134→				if err != nil {
   135→					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
   136→					atomic.AddInt32(&errs, 1)
   137→					continue
   138→				}
   139→
   140→				for _, checkpoint := range files {
   141→					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
   142→						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
   143→						atomic.AddInt32(&errs, 1)
   144→					} else {
   145→						atomic.AddInt32(&n, 1)
   146→					}
   147→				}
   148→			}
   149→		}()
   150→	}
   151→
   152→	for _, de1 := range entries1 {
   153→		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
   154→		if e != nil {
   155→			err = e
   156→			continue
   157→		}
   158→
   159→		for _, de2 := range entries2 {
   160→			work <- workItem{
   161→				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
   162→				cluster: de1.Name(),
   163→				host:    de2.Name(),
   164→			}
   165→		}
   166→	}
   167→
   168→	close(work)
   169→	wg.Wait()
   170→
   171→	if err != nil {
   172→		return int(n), err
   173→	}
   174→	if errs > 0 {
   175→		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
   176→	}
   177→	return int(n), nil
   178→}
   179→
   180→// archiveCheckpoints archives checkpoint files to Parquet format.
   181→// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
   182→// Workers load checkpoint files from disk and send CheckpointFile trees on a
   183→// back-pressured channel. The main thread streams each tree directly to Parquet
   184→// rows without materializing all rows in memory.
   185→func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
   186→	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
   187→	startTime := time.Now()
   188→
   189→	clusterEntries, err := os.ReadDir(checkpointsDir)
   190→	if err != nil {
   191→		return 0, err
   192→	}
   193→
   194→	totalFiles := 0
   195→	var clusterErrors []string
   196→
   197→	for _, clusterEntry := range clusterEntries {
   198→		if !clusterEntry.IsDir() {
   199→			continue
   200→		}
   201→
   202→		cluster := clusterEntry.Name()
   203→		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
   204→		if err != nil {
   205→			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
   206→			clusterErrors = append(clusterErrors, cluster)
   207→			continue
   208→		}
   209→
   210→		// Workers load checkpoint files from disk; main thread writes to parquet.
   211→		type hostResult struct {
   212→			checkpoints []*CheckpointFile
   213→			hostname    string
   214→			files       []string // checkpoint filenames to delete after successful write
   215→			dir         string   // checkpoint directory for this host
   216→		}
   217→
   218→		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
   219→		results := make(chan hostResult, 2)
   220→		work := make(chan struct {
   221→			dir, host string
   222→		}, Keys.NumWorkers)
   223→
   224→		var wg sync.WaitGroup
   225→		errs := int32(0)
   226→
   227→		wg.Add(Keys.NumWorkers)
   228→		for w := 0; w < Keys.NumWorkers; w++ {
   229→			go func() {
   230→				defer wg.Done()
   231→				for item := range work {
   232→					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
   233→					if err != nil {
   234→						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
   235→						atomic.AddInt32(&errs, 1)
   236→						continue
   237→					}
   238→					if len(checkpoints) > 0 {
   239→						results <- hostResult{
   240→							checkpoints: checkpoints,
   241→							hostname:    item.host,
   242→							files:       files,
   243→							dir:         item.dir,
   244→						}
   245→					}
   246→				}
   247→			}()
   248→		}
   249→
   250→		go func() {
   251→			for _, hostEntry := range hostEntries {
   252→				if !hostEntry.IsDir() {
   253→					continue
   254→				}
   255→				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
   256→				work <- struct {
   257→					dir, host string
   258→				}{dir: dir, host: hostEntry.Name()}
   259→			}
   260→			close(work)
   261→			wg.Wait()
   262→			close(results)
   263→		}()
   264→
   265→		// Open streaming writer and write each host's checkpoint files as a row group
   266→		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
   267→		writer, err := newParquetArchiveWriter(parquetFile)
   268→		if err != nil {
   269→			// Drain results channel to unblock workers
   270→			for range results {
   271→			}
   272→			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
   273→			clusterErrors = append(clusterErrors, cluster)
   274→			continue
   275→		}
   276→
   277→		type deleteItem struct {
   278→			dir   string
   279→			files []string
   280→		}
   281→		var toDelete []deleteItem
   282→		writeErr := error(nil)
   283→
   284→		for r := range results {
   285→			if writeErr == nil {
   286→				// Stream each checkpoint file directly to parquet rows.
   287→				// Each checkpoint is processed and discarded before the next.
   288→				for _, cf := range r.checkpoints {
   289→					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
   290→						writeErr = err
   291→						break
   292→					}
   293→				}
   294→				// Flush once per host to keep row group count within parquet limits.
   295→				if writeErr == nil {
   296→					if err := writer.FlushRowGroup(); err != nil {
   297→						writeErr = err
   298→					}
   299→				}
   300→			}
   301→			// Always track files for deletion (even if write failed, we still drain)
   302→			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
   303→		}
   304→
   305→		if err := writer.Close(); err != nil && writeErr == nil {
   306→			writeErr = err
   307→		}
   308→
   309→		if errs > 0 {
   310→			cclog.Errorf("[METRICSTORE]> %d errors reading checkpoints for cluster %s", errs, cluster)
   311→			clusterErrors = append(clusterErrors, cluster)
   312→			os.Remove(parquetFile)
   313→			continue
   314→		}
   315→
   316→		if writer.count == 0 {
   317→			// No data written — remove empty file
   318→			os.Remove(parquetFile)
   319→			continue
   320→		}
   321→
   322→		if writeErr != nil {
   323→			os.Remove(parquetFile)
   324→			cclog.Errorf("[METRICSTORE]> error writing parquet archive for cluster %s: %s", cluster, writeErr.Error())
   325→			clusterErrors = append(clusterErrors, cluster)
   326→			continue
   327→		}
   328→
   329→		// Delete archived checkpoint files
   330→		for _, item := range toDelete {
   331→			for _, file := range item.files {
   332→				filename := filepath.Join(item.dir, file)
   333→				if err := os.Remove(filename); err != nil {
   334→					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
   335→				} else {
   336→					totalFiles++
   337→				}
   338→			}
   339→		}
   340→
   341→		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
   342→			writer.count, totalFiles, cluster, parquetFile)
   343→	}
   344→
   345→	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
   346→
   347→	if len(clusterErrors) > 0 {
   348→		return totalFiles, fmt.Errorf("archiving failed for clusters: %s", strings.Join(clusterErrors, ", "))
   349→	}
   350→
   351→	return totalFiles, nil
   352→}
   353→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides level.go: Hierarchical tree structure for metric storage.
     7→//
     8→// # Level Architecture
     9→//
    10→// The Level type forms a tree structure where each node represents a level in the
    11→// ClusterCockpit hierarchy: cluster → host → socket → core → hwthread, with special
    12→// nodes for memory domains and accelerators.
    13→//
    14→// Structure:
    15→//
    16→//	Root Level (cluster="emmy")
    17→//	├─ Level (host="node001")
    18→//	│  ├─ Level (socket="0")
    19→//	│  │  ├─ Level (core="0") [stores cpu0 metrics]
    20→//	│  │  └─ Level (core="1") [stores cpu1 metrics]
    21→//	│  └─ Level (socket="1")
    22→//	│     └─ ...
    23→//	└─ Level (host="node002")
    24→//	   └─ ...
    25→//
    26→// Each Level can:
    27→//   - Hold data (metrics slice of buffer pointers)
    28→//   - Have child nodes (children map[string]*Level)
    29→//   - Both simultaneously (inner nodes can store aggregated metrics)
    30→//
    31→// # Selector Paths
    32→//
    33→// Selectors are hierarchical paths: []string{"cluster", "host", "component"}.
    34→// Example: []string{"emmy", "node001", "cpu0"} navigates to the cpu0 core level.
    35→//
    36→// # Concurrency
    37→//
    38→// RWMutex protects children map and metrics slice. Read-heavy workload (metric reads)
    39→// uses RLock. Writes (new levels, buffer updates) use Lock. Double-checked locking
    40→// prevents races during level creation.
    41→package metricstore
    42→
    43→import (
    44→	"sync"
    45→	"time"
    46→	"unsafe"
    47→
    48→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    49→	"github.com/ClusterCockpit/cc-lib/v2/util"
    50→)
    51→
    52→// Level represents a node in the hierarchical metric storage tree.
    53→//
    54→// Can be both a leaf or inner node. Inner nodes hold data in 'metrics' for aggregated
    55→// values (e.g., socket-level metrics derived from core-level data). Named "Level"
    56→// instead of "node" to avoid confusion with cluster nodes (hosts).
    57→//
    58→// Fields:
    59→//   - children: Map of child level names to Level pointers (e.g., "cpu0" → Level)
    60→//   - metrics:  Slice of buffer pointers (one per metric, indexed by MetricConfig.offset)
    61→//   - lock:     RWMutex for concurrent access (read-heavy, write-rare)
    62→type Level struct {
    63→	children map[string]*Level
    64→	metrics  []*buffer
    65→	lock     sync.RWMutex
    66→}
    67→
    68→// findLevelOrCreate navigates to or creates the level specified by selector.
    69→//
    70→// Recursively descends the tree, creating missing levels as needed. Uses double-checked
    71→// locking: RLock first (fast path), then Lock if creation needed (slow path), then
    72→// re-check after acquiring Lock to handle races.
    73→//
    74→// Example selector: []string{"emmy", "node001", "cpu0"}
    75→// Navigates: root → emmy → node001 → cpu0, creating levels as needed.
    76→//
    77→// Parameters:
    78→//   - selector: Hierarchical path (consumed recursively, decreasing depth)
    79→//   - nMetrics: Number of metric slots to allocate in new levels
    80→//
    81→// Returns:
    82→//   - *Level: The target level (existing or newly created)
    83→//
    84→// Note: sync.Map may improve performance for high-concurrency writes, but current
    85→// approach suffices for read-heavy workload.
    86→func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
    87→	if len(selector) == 0 {
    88→		return l
    89→	}
    90→
    91→	// Allow concurrent reads:
    92→	l.lock.RLock()
    93→	var child *Level
    94→	var ok bool
    95→	if l.children == nil {
    96→		// Children map needs to be created...
    97→		l.lock.RUnlock()
    98→	} else {
    99→		child, ok = l.children[selector[0]]
   100→		l.lock.RUnlock()
   101→		if ok {
   102→			return child.findLevelOrCreate(selector[1:], nMetrics)
   103→		}
   104→	}
   105→
   106→	// The level does not exist, take write lock for unique access:
   107→	l.lock.Lock()
   108→	// While this thread waited for the write lock, another thread
   109→	// could have created the child node.
   110→	if l.children != nil {
   111→		child, ok = l.children[selector[0]]
   112→		if ok {
   113→			l.lock.Unlock()
   114→			return child.findLevelOrCreate(selector[1:], nMetrics)
   115→		}
   116→	}
   117→
   118→	child = &Level{
   119→		metrics:  make([]*buffer, nMetrics),
   120→		children: nil,
   121→	}
   122→
   123→	if l.children != nil {
   124→		l.children[selector[0]] = child
   125→	} else {
   126→		l.children = map[string]*Level{selector[0]: child}
   127→	}
   128→	l.lock.Unlock()
   129→	return child.findLevelOrCreate(selector[1:], nMetrics)
   130→}
   131→
   132→// collectPaths gathers all selector paths at the specified depth in the tree.
   133→//
   134→// Recursively traverses children, collecting paths when currentDepth+1 == targetDepth.
   135→// Each path is a selector that can be used with findLevel() or findBuffers().
   136→//
   137→// Explicitly copies slices to avoid shared underlying arrays between siblings, preventing
   138→// unintended mutations.
   139→//
   140→// Parameters:
   141→//   - currentDepth: Depth of current level (0 = root)
   142→//   - targetDepth:  Depth to collect paths from
   143→//   - currentPath:  Path accumulated so far
   144→//   - results:      Output slice (appended to)
   145→//
   146→// Example: collectPaths(0, 2, []string{}, &results) collects all 2-level paths
   147→// like []string{"emmy", "node001"}, []string{"emmy", "node002"}, etc.
   148→func (l *Level) collectPaths(currentDepth, targetDepth int, currentPath []string, results *[][]string) {
   149→	l.lock.RLock()
   150→	defer l.lock.RUnlock()
   151→
   152→	for key, child := range l.children {
   153→		if child == nil {
   154→			continue
   155→		}
   156→
   157→		// We explicitly make a new slice and copy data to avoid sharing underlying arrays between siblings
   158→		newPath := make([]string, len(currentPath))
   159→		copy(newPath, currentPath)
   160→		newPath = append(newPath, key)
   161→
   162→		// Check depth, and just return if depth reached
   163→		if currentDepth+1 == targetDepth {
   164→			*results = append(*results, newPath)
   165→		} else {
   166→			child.collectPaths(currentDepth+1, targetDepth, newPath, results)
   167→		}
   168→	}
   169→}
   170→
   171→// free removes buffers older than the retention threshold from the entire subtree.
   172→//
   173→// Recursively frees buffers in this level's metrics and all child levels. Buffers
   174→// with standard capacity (BufferCap) are returned to the pool. Called by the
   175→// retention worker to enforce retention policies.
   176→//
   177→// Parameters:
   178→//   - t: Retention threshold timestamp (Unix seconds)
   179→//
   180→// Returns:
   181→//   - int:   Total number of buffers freed in this subtree
   182→//   - error: Non-nil on failure (propagated from children)
   183→func (l *Level) free(t int64) (int, error) {
   184→	n, _, err := l.freeAndCheckEmpty(t)
   185→	return n, err
   186→}
   187→
   188→// freeAndCheckEmpty performs free() and atomically checks if the level is empty
   189→// while still holding its own lock, avoiding a TOCTOU race between free() and
   190→// a separate isEmpty() call.
   191→func (l *Level) freeAndCheckEmpty(t int64) (int, bool, error) {
   192→	l.lock.Lock()
   193→	defer l.lock.Unlock()
   194→
   195→	n := 0
   196→	for i, b := range l.metrics {
   197→		if b != nil {
   198→			delme, m := b.free(t)
   199→			n += m
   200→			if delme {
   201→				if cap(b.data) != BufferCap {
   202→					b.data = make([]schema.Float, 0, BufferCap)
   203→				}
   204→				b.lastUsed = time.Now().Unix()
   205→				bufferPool.Put(b)
   206→				l.metrics[i] = nil
   207→			}
   208→		}
   209→	}
   210→
   211→	for key, child := range l.children {
   212→		m, empty, err := child.freeAndCheckEmpty(t)
   213→		n += m
   214→		if err != nil {
   215→			return n, false, err
   216→		}
   217→		if empty {
   218→			delete(l.children, key)
   219→		}
   220→	}
   221→
   222→	// Check emptiness while still holding the lock
   223→	empty := len(l.children) == 0
   224→	if empty {
   225→		for _, b := range l.metrics {
   226→			if b != nil {
   227→				empty = false
   228→				break
   229→			}
   230→		}
   231→	}
   232→
   233→	return n, empty, nil
   234→}
   235→
   236→// forceFree removes the oldest buffer from each metric chain in the subtree.
   237→//
   238→// Unlike free(), which removes based on time threshold, this unconditionally removes
   239→// the oldest buffer in each chain. Used by MemoryUsageTracker when memory cap is
   240→// exceeded and time-based retention is insufficient.
   241→//
   242→// Recursively processes current level's metrics and all child levels.
   243→//
   244→// Returns:
   245→//   - int:   Total number of buffers freed in this subtree
   246→//   - error: Non-nil on failure (propagated from children)
   247→func (l *Level) forceFree() (int, error) {
   248→	l.lock.Lock()
   249→	defer l.lock.Unlock()
   250→
   251→	n := 0
   252→
   253→	// Iterate over metrics in the current level
   254→	for i, b := range l.metrics {
   255→		if b != nil {
   256→			// Attempt to free the oldest buffer in this chain
   257→			delme, freedCount := b.forceFreeOldest()
   258→			n += freedCount
   259→
   260→			// If delme is true, it means 'b' itself (the head) was the oldest
   261→			// and needs to be removed from the slice.
   262→			if delme {
   263→				b.next = nil
   264→				b.prev = nil
   265→				if cap(b.data) != BufferCap {
   266→					b.data = make([]schema.Float, 0, BufferCap)
   267→				}
   268→				b.lastUsed = time.Now().Unix()
   269→				bufferPool.Put(b)
   270→				l.metrics[i] = nil
   271→			}
   272→		}
   273→	}
   274→
   275→	// Recursively traverse children
   276→	for _, child := range l.children {
   277→		m, err := child.forceFree()
   278→		n += m
   279→		if err != nil {
   280→			return n, err
   281→		}
   282→	}
   283→
   284→	return n, nil
   285→}
   286→
   287→// sizeInBytes calculates the total memory usage of all buffers in the subtree.
   288→//
   289→// Recursively sums buffer data sizes (count of Float values × sizeof(Float)) across
   290→// this level's metrics and all child levels. Used by MemoryUsageTracker to enforce
   291→// memory cap limits.
   292→//
   293→// Returns:
   294→//   - int64: Total bytes used by buffer data in this subtree
   295→func (l *Level) sizeInBytes() int64 {
   296→	l.lock.RLock()
   297→	defer l.lock.RUnlock()
   298→	size := int64(0)
   299→
   300→	for _, b := range l.metrics {
   301→		if b != nil {
   302→			size += b.count() * int64(unsafe.Sizeof(schema.Float(0)))
   303→			size += b.bufferCount() * int64(unsafe.Sizeof(buffer{}))
   304→		}
   305→	}
   306→
   307→	for _, child := range l.children {
   308→		size += child.sizeInBytes()
   309→	}
   310→
   311→	return size
   312→}
   313→
   314→// findLevel navigates to the level specified by selector, returning nil if not found.
   315→//
   316→// Read-only variant of findLevelOrCreate. Does not create missing levels.
   317→// Recursively descends the tree following the selector path.
   318→//
   319→// Parameters:
   320→//   - selector: Hierarchical path (e.g., []string{"emmy", "node001", "cpu0"})
   321→//
   322→// Returns:
   323→//   - *Level: The target level, or nil if any component in the path does not exist
   324→func (l *Level) findLevel(selector []string) *Level {
   325→	if len(selector) == 0 {
   326→		return l
   327→	}
   328→
   329→	l.lock.RLock()
   330→	defer l.lock.RUnlock()
   331→
   332→	lvl := l.children[selector[0]]
   333→	if lvl == nil {
   334→		return nil
   335→	}
   336→
   337→	return lvl.findLevel(selector[1:])
   338→}
   339→
   340→// findBuffers invokes callback on all buffers matching the selector pattern.
   341→//
   342→// Supports flexible selector patterns (from cc-lib/util.Selector):
   343→//   - Exact match: Selector element with String set (e.g., "node001")
   344→//   - Group match: Selector element with Group set (e.g., ["cpu0", "cpu2", "cpu4"])
   345→//   - Wildcard:    Selector element with Any=true (matches all children)
   346→//
   347→// Empty selector (len==0) matches current level's buffer at 'offset' and recursively
   348→// all descendant buffers at the same offset (used for aggregation queries).
   349→//
   350→// Parameters:
   351→//   - selector: Pattern to match (consumed recursively)
   352→//   - offset:   Metric index in metrics slice (from MetricConfig.offset)
   353→//   - f:        Callback invoked on each matching buffer
   354→//
   355→// Returns:
   356→//   - error: First error returned by callback, or nil if all succeeded
   357→//
   358→// Example:
   359→//
   360→//	// Find all cpu0 buffers across all hosts:
   361→//	findBuffers([]Selector{{Any: true}, {String: "cpu0"}}, metricOffset, callback)
   362→func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
   363→	l.lock.RLock()
   364→	defer l.lock.RUnlock()
   365→
   366→	if len(selector) == 0 {
   367→		b := l.metrics[offset]
   368→		if b != nil {
   369→			return f(b)
   370→		}
   371→
   372→		for _, lvl := range l.children {
   373→			err := lvl.findBuffers(nil, offset, f)
   374→			if err != nil {
   375→				return err
   376→			}
   377→		}
   378→		return nil
   379→	}
   380→
   381→	sel := selector[0]
   382→	if len(sel.String) != 0 && l.children != nil {
   383→		lvl, ok := l.children[sel.String]
   384→		if ok {
   385→			err := lvl.findBuffers(selector[1:], offset, f)
   386→			if err != nil {
   387→				return err
   388→			}
   389→		}
   390→		return nil
   391→	}
   392→
   393→	if sel.Group != nil && l.children != nil {
   394→		for _, key := range sel.Group {
   395→			lvl, ok := l.children[key]
   396→			if ok {
   397→				err := lvl.findBuffers(selector[1:], offset, f)
   398→				if err != nil {
   399→					return err
   400→				}
   401→			}
   402→		}
   403→		return nil
   404→	}
   405→
   406→	if sel.Any && l.children != nil {
   407→		for _, lvl := range l.children {
   408→			if err := lvl.findBuffers(selector[1:], offset, f); err != nil {
   409→				return err
   410→			}
   411→		}
   412→		return nil
   413→	}
   414→
   415→	return nil
   416→}
   417→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>