Update the doc comments in metricstore package to reflect the current implementation


Let me explore the metricstore package to understand the current implementation and then update the doc comments accordingly.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides an efficient in-memory time-series metric storage system
     7→// with support for hierarchical data organization, checkpointing, and archiving.
     8→//
     9→// The package organizes metrics in a tree structure (cluster → host → component) and
    10→// provides concurrent read/write access to metric data with configurable aggregation strategies.
    11→// Background goroutines handle periodic checkpointing (JSON or Avro format), archiving old data,
    12→// and enforcing retention policies.
    13→//
    14→// Key features:
    15→//   - In-memory metric storage with configurable retention
    16→//   - Hierarchical data organization (selectors)
    17→//   - Concurrent checkpoint/archive workers
    18→//   - Support for sum and average aggregation
    19→//   - NATS integration for metric ingestion
    20→package metricstore
    21→
    22→import (
    23→	"bytes"
    24→	"context"
    25→	"encoding/json"
    26→	"errors"
    27→	"fmt"
    28→	"runtime"
    29→	"runtime/debug"
    30→	"slices"
    31→	"sync"
    32→	"time"
    33→
    34→	"github.com/ClusterCockpit/cc-backend/internal/config"
    35→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    36→	"github.com/ClusterCockpit/cc-lib/v2/resampler"
    37→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    38→	"github.com/ClusterCockpit/cc-lib/v2/util"
    39→)
    40→
    41→// GlobalState holds the global state for the metric store with thread-safe access.
    42→type GlobalState struct {
    43→	mu                sync.RWMutex
    44→	lastRetentionTime int64
    45→	selectorsExcluded bool
    46→}
    47→
    48→var (
    49→	singleton  sync.Once
    50→	msInstance *MemoryStore
    51→	// shutdownFunc stores the context cancellation function created in Init
    52→	// and is called during Shutdown to cancel all background goroutines
    53→	shutdownFunc   context.CancelFunc
    54→	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
    55→	// Create a global instance
    56→	state = &GlobalState{}
    57→)
    58→
    59→// NodeProvider provides information about nodes currently in use by running jobs.
    60→//
    61→// This interface allows metricstore to query job information without directly
    62→// depending on the repository package, breaking the import cycle.
    63→//
    64→// Implementations should return nodes that are actively processing jobs started
    65→// before the given timestamp. These nodes will be excluded from retention-based
    66→// garbage collection to prevent data loss for jobs that are still running or
    67→// recently completed.
    68→type NodeProvider interface {
    69→	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
    70→	// that are currently in use by jobs that started before the given timestamp.
    71→	//
    72→	// Parameters:
    73→	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
    74→	//
    75→	// Returns:
    76→	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
    77→	//   - Error if the query fails
    78→	GetUsedNodes(ts int64) (map[string][]string, error)
    79→}
    80→
    81→// Metric represents a single metric data point to be written to the store.
    82→type Metric struct {
    83→	Name  string
    84→	Value schema.Float
    85→	// MetricConfig contains frequency and aggregation settings for this metric.
    86→	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
    87→	MetricConfig MetricConfig
    88→}
    89→
    90→// MemoryStore is the main in-memory time-series metric storage implementation.
    91→//
    92→// It organizes metrics in a hierarchical tree structure where each level represents
    93→// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
    94→// store multiple metrics as time-series buffers.
    95→//
    96→// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
    97→// All public methods are safe for concurrent use.
    98→type MemoryStore struct {
    99→	Metrics      map[string]MetricConfig
   100→	root         Level
   101→	nodeProvider NodeProvider
   102→}
   103→
   104→// Init initializes the metric store from configuration and starts background workers.
   105→//
   106→// This function must be called exactly once before any other metricstore operations.
   107→// It performs the following initialization steps:
   108→//  1. Validates and decodes the metric store configuration
   109→//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
   110→//  3. Loads metric configurations from all registered clusters
   111→//  4. Restores checkpoints within the retention window
   112→//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
   113→//  6. Optionally subscribes to NATS for real-time metric ingestion
   114→//
   115→// Parameters:
   116→//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig)
   117→//   - wg: WaitGroup that will be incremented for each background goroutine started
   118→//
   119→// The function will call cclog.Fatal on critical errors during initialization.
   120→// Use Shutdown() to cleanly stop all background workers started by Init().
   121→//
   122→// Note: Signal handling must be implemented by the caller. Call Shutdown() when
   123→// receiving termination signals to ensure checkpoint data is persisted.
   124→func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
   125→	startupTime := time.Now()
   126→
   127→	if rawConfig != nil {
   128→		config.Validate(configSchema, rawConfig)
   129→		dec := json.NewDecoder(bytes.NewReader(rawConfig))
   130→		dec.DisallowUnknownFields()
   131→		if err := dec.Decode(&Keys); err != nil {
   132→			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
   133→		}
   134→	}
   135→
   136→	// Set NumWorkers from config or use default
   137→	if Keys.NumWorkers <= 0 {
   138→		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
   139→	}
   140→	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
   141→
   142→	// Pass the config.MetricStoreKeys
   143→	InitMetrics(metrics)
   144→
   145→	ms := GetMemoryStore()
   146→
   147→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   148→	if err != nil {
   149→		cclog.Fatal(err)
   150→	}
   151→
   152→	restoreFrom := startupTime.Add(-d)
   153→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   154→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   155→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   156→	if err != nil {
   157→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   158→	} else {
   159→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   160→	}
   161→
   162→	// Try to use less memory by forcing a GC run here and then
   163→	// lowering the target percentage. The default of 100 means
   164→	// that only once the ratio of new allocations execeds the
   165→	// previously active heap, a GC is triggered.
   166→	// Forcing a GC here will set the "previously active heap"
   167→	// to a minumum.
   168→	// runtime.GC()
   169→
   170→	ctx, shutdown := context.WithCancel(context.Background())
   171→
   172→	Retention(wg, ctx)
   173→	Checkpointing(wg, ctx)
   174→	CleanUp(wg, ctx)
   175→	WALStaging(wg, ctx)
   176→	MemoryUsageTracker(wg, ctx)
   177→
   178→	// Note: Signal handling has been removed from this function.
   179→	// The caller is responsible for handling shutdown signals and calling
   180→	// the shutdown() function when appropriate.
   181→	// Store the shutdown function for later use by Shutdown()
   182→	shutdownFuncMu.Lock()
   183→	shutdownFunc = shutdown
   184→	shutdownFuncMu.Unlock()
   185→
   186→	if Keys.Subscriptions != nil {
   187→		wg.Go(func() {
   188→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   189→				cclog.Fatal(err)
   190→			}
   191→		})
   192→	}
   193→}
   194→
   195→// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
   196→//
   197→// This function must be called before GetMemoryStore() and can only be called once due to
   198→// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
   199→//
   200→// Parameters:
   201→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
   202→//
   203→// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
   204→//
   205→// After this call, the global msInstance is ready for use via GetMemoryStore().
   206→func InitMetrics(metrics map[string]MetricConfig) {
   207→	singleton.Do(func() {
   208→		offset := 0
   209→		for key, cfg := range metrics {
   210→			if cfg.Frequency == 0 {
   211→				panic("[METRICSTORE]> invalid frequency")
   212→			}
   213→
   214→			metrics[key] = MetricConfig{
   215→				Frequency:   cfg.Frequency,
   216→				Aggregation: cfg.Aggregation,
   217→				offset:      offset,
   218→			}
   219→			offset += 1
   220→		}
   221→
   222→		msInstance = &MemoryStore{
   223→			root: Level{
   224→				metrics:  make([]*buffer, len(metrics)),
   225→				children: make(map[string]*Level),
   226→			},
   227→			Metrics: metrics,
   228→		}
   229→	})
   230→}
   231→
   232→// GetMemoryStore returns the singleton MemoryStore instance.
   233→//
   234→// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
   235→//
   236→// This function is safe for concurrent use after initialization.
   237→func GetMemoryStore() *MemoryStore {
   238→	if msInstance == nil {
   239→		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
   240→	}
   241→
   242→	return msInstance
   243→}
   244→
   245→func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
   246→	if metric, ok := ms.Metrics[metricName]; ok {
   247→		return metric.Frequency, nil
   248→	}
   249→	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
   250→}
   251→
   252→// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
   253→// This must be called during initialization to provide job state information
   254→// for selective buffer retention during Free operations.
   255→// If not set, the Free function will fall back to freeing all buffers.
   256→func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
   257→	ms.nodeProvider = provider
   258→}
   259→
   260→// Shutdown performs a graceful shutdown of the metric store.
   261→//
   262→// This function cancels all background goroutines started by Init() and writes
   263→// a final checkpoint to disk before returning. It should be called when the
   264→// application receives a termination signal.
   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	totalStart := time.Now()
   275→
   276→	shutdownFuncMu.Lock()
   277→	defer shutdownFuncMu.Unlock()
   278→	if shutdownFunc != nil {
   279→		shutdownFunc()
   280→	}
   281→	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))
   282→
   283→	if Keys.Checkpoints.FileFormat == "wal" {
   284→		// Signal producers to stop sending before closing channels,
   285→		// preventing send-on-closed-channel panics from in-flight NATS workers.
   286→		walShuttingDown.Store(true)
   287→		// Brief grace period for in-flight DecodeLine calls to complete.
   288→		time.Sleep(100 * time.Millisecond)
   289→
   290→		for _, ch := range walShardChs {
   291→			close(ch)
   292→		}
   293→		drainStart := time.Now()
   294→		WaitForWALStagingDrain()
   295→		cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
   296→	}
   297→
   298→	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
   299→	checkpointStart := time.Now()
   300→	var files int
   301→	var err error
   302→
   303→	ms := GetMemoryStore()
   304→
   305→	lastCheckpointMu.Lock()
   306→	from := lastCheckpoint
   307→	lastCheckpointMu.Unlock()
   308→
   309→	if Keys.Checkpoints.FileFormat == "wal" {
   310→		// WAL files are deleted per-host inside ToCheckpointWAL workers.
   311→		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   312→	} else {
   313→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   314→	}
   315→
   316→	if err != nil {
   317→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
   318→	}
   319→	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
   320→}
   321→
   322→// Retention starts a background goroutine that periodically frees old metric data.
   323→//
   324→// This worker runs at half the retention interval and calls Free() to remove buffers
   325→// older than the configured retention time. It respects the NodeProvider to preserve
   326→// data for nodes with active jobs.
   327→//
   328→// Parameters:
   329→//   - wg: WaitGroup to signal completion when context is cancelled
   330→//   - ctx: Context for cancellation signal
   331→//
   332→// The goroutine exits when ctx is cancelled.
   333→func Retention(wg *sync.WaitGroup, ctx context.Context) {
   334→	ms := GetMemoryStore()
   335→
   336→	wg.Go(func() {
   337→		d, err := time.ParseDuration(Keys.RetentionInMemory)
   338→		if err != nil {
   339→			cclog.Fatal(err)
   340→		}
   341→		if d <= 0 {
   342→			return
   343→		}
   344→
   345→		tickInterval := d / 2
   346→		if tickInterval <= 0 {
   347→			return
   348→		}
   349→		ticker := time.NewTicker(tickInterval)
   350→		defer ticker.Stop()
   351→
   352→		for {
   353→			select {
   354→			case <-ctx.Done():
   355→				return
   356→			case <-ticker.C:
   357→				state.mu.Lock()
   358→
   359→				t := time.Now().Add(-d)
   360→
   361→				state.lastRetentionTime = t.Unix()
   362→
   363→				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
   364→
   365→				freed, err := Free(ms, t)
   366→				if err != nil {
   367→					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
   368→				} else {
   369→					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
   370→				}
   371→
   372→				state.mu.Unlock()
   373→
   374→				// Clean up the buffer pool
   375→				bufferPool.Clean(state.lastRetentionTime)
   376→			}
   377→		}
   378→	})
   379→}
   380→
   381→// MemoryUsageTracker starts a background goroutine that monitors memory usage.
   382→//
   383→// This worker checks actual process memory usage (via runtime.MemStats) periodically
   384→// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
   385→// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
   386→// performance issues.
   387→//
   388→// The tracker logs both actual memory usage (heap allocated) and metric data size for
   389→// visibility into memory overhead from Go runtime structures and allocations.
   390→//
   391→// Parameters:
   392→//   - wg: WaitGroup to signal completion when context is cancelled
   393→//   - ctx: Context for cancellation signal
   394→//
   395→// The goroutine exits when ctx is cancelled.
   396→func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
   397→	ms := GetMemoryStore()
   398→
   399→	wg.Go(func() {
   400→		normalInterval := DefaultMemoryUsageTrackerInterval
   401→		fastInterval := 30 * time.Second
   402→
   403→		if normalInterval <= 0 {
   404→			return
   405→		}
   406→
   407→		ticker := time.NewTicker(normalInterval)
   408→		defer ticker.Stop()
   409→		currentInterval := normalInterval
   410→
   411→		for {
   412→			select {
   413→			case <-ctx.Done():
   414→				return
   415→			case <-ticker.C:
   416→				var mem runtime.MemStats
   417→				runtime.ReadMemStats(&mem)
   418→				actualMemoryGB := float64(mem.Alloc) / 1e9
   419→				metricDataGB := ms.SizeInGB()
   420→				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
   421→
   422→				freedExcluded := 0
   423→				freedEmergency := 0
   424→				var err error
   425→
   426→				state.mu.RLock()
   427→				lastRetention := state.lastRetentionTime
   428→				selectorsExcluded := state.selectorsExcluded
   429→				state.mu.RUnlock()
   430→
   431→				if lastRetention != 0 && selectorsExcluded {
   432→					freedExcluded, err = ms.Free(nil, lastRetention)
   433→					if err != nil {
   434→						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
   435→					}
   436→
   437→					if freedExcluded > 0 {
   438→						debug.FreeOSMemory()
   439→						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
   440→					}
   441→				}
   442→
   443→				runtime.ReadMemStats(&mem)
   444→				actualMemoryGB = float64(mem.Alloc) / 1e9
   445→
   446→				if actualMemoryGB > float64(Keys.MemoryCap) {
   447→					cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)
   448→
   449→					// Use progressive time-based Free with increasing threshold
   450→					// instead of ForceFree loop — fewer tree traversals, more effective
   451→					d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
   452→					if parseErr != nil {
   453→						cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
   454→					} else {
   455→						thresholds := []float64{0.75, 0.5, 0.25}
   456→						for _, fraction := range thresholds {
   457→							threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
   458→							freed, freeErr := ms.Free(nil, threshold.Unix())
   459→							if freeErr != nil {
   460→								cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
   461→							}
   462→							freedEmergency += freed
   463→
   464→							bufferPool.Clear()
   465→							runtime.GC()
   466→							runtime.ReadMemStats(&mem)
   467→							actualMemoryGB = float64(mem.Alloc) / 1e9
   468→
   469→							if actualMemoryGB < float64(Keys.MemoryCap) {
   470→								break
   471→							}
   472→						}
   473→					}
   474→
   475→					bufferPool.Clear()
   476→					debug.FreeOSMemory()
   477→
   478→					runtime.ReadMemStats(&mem)
   479→					actualMemoryGB = float64(mem.Alloc) / 1e9
   480→
   481→					if actualMemoryGB >= float64(Keys.MemoryCap) {
   482→						cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
   483→					} else {
   484→						cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
   485→					}
   486→				}
   487→
   488→				// Adaptive ticker: check more frequently when memory is high
   489→				memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
   490→				if memoryRatio > 0.8 && currentInterval != fastInterval {
   491→					ticker.Reset(fastInterval)
   492→					currentInterval = fastInterval
   493→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
   494→				} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
   495→					ticker.Reset(normalInterval)
   496→					currentInterval = normalInterval
   497→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
   498→				}
   499→			}
   500→		}
   501→	})
   502→}
   503→
   504→// Free removes metric data older than the given time while preserving data for active nodes.
   505→//
   506→// This function implements intelligent retention by consulting the NodeProvider (if configured)
   507→// to determine which nodes are currently in use by running jobs. Data for these nodes is
   508→// preserved even if older than the retention time.
   509→//
   510→// Parameters:
   511→//   - ms: The MemoryStore instance
   512→//   - t: Time threshold - buffers with data older than this will be freed
   513→//
   514→// Returns:
   515→//   - Number of buffers freed
   516→//   - Error if NodeProvider query fails
   517→//
   518→// Behavior:
   519→//   - If no NodeProvider is set: frees all buffers older than t
   520→//   - If NodeProvider returns empty map: frees all buffers older than t
   521→//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
   522→func Free(ms *MemoryStore, t time.Time) (int, error) {
   523→	// If no NodeProvider is configured, free all buffers older than t
   524→	if ms.nodeProvider == nil {
   525→		return ms.Free(nil, t.Unix())
   526→	}
   527→
   528→	excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
   529→	if err != nil {
   530→		return 0, err
   531→	}
   532→
   533→	switch lenMap := len(excludeSelectors); lenMap {
   534→
   535→	// If the length of the map returned by GetUsedNodes() is 0,
   536→	// then use default Free method with nil selector
   537→	case 0:
   538→		state.selectorsExcluded = false
   539→		return ms.Free(nil, t.Unix())
   540→
   541→	// Else formulate selectors, exclude those from the map
   542→	// and free the rest of the selectors
   543→	default:
   544→		state.selectorsExcluded = true
   545→		selectors := GetSelectors(ms, excludeSelectors)
   546→		return FreeSelected(ms, selectors, t)
   547→	}
   548→}
   549→
   550→// FreeSelected frees buffers for specific selectors while preserving others.
   551→//
   552→// This function is used when we want to retain some specific nodes beyond the retention time.
   553→// It iterates through the provided selectors and frees their associated buffers.
   554→//
   555→// Parameters:
   556→//   - ms: The MemoryStore instance
   557→//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
   558→//   - t: Time threshold for freeing buffers
   559→//
   560→// Returns the total number of buffers freed and any error encountered.
   561→func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
   562→	freed := 0
   563→
   564→	for _, selector := range selectors {
   565→
   566→		freedBuffers, err := ms.Free(selector, t.Unix())
   567→		if err != nil {
   568→			cclog.Errorf("error while freeing selected buffers: %#v", err)
   569→		}
   570→		freed += freedBuffers
   571→
   572→	}
   573→
   574→	return freed, nil
   575→}
   576→
   577→// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
   578→//
   579→// This function generates a list of selectors whose buffers should be freed by excluding
   580→// selectors that correspond to nodes currently in use by running jobs.
   581→//
   582→// Parameters:
   583→//   - ms: The MemoryStore instance
   584→//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
   585→//
   586→// Returns a list of selectors ([]string paths) that can be safely freed.
   587→//
   588→// Example:
   589→//
   590→//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
   591→//	and excludeSelectors contains {"emmy": ["node001"]},
   592→//	then only [["emmy", "node002"]] is returned.
   593→func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
   594→	allSelectors := ms.GetPaths(2)
   595→
   596→	filteredSelectors := make([][]string, 0, len(allSelectors))
   597→
   598→	for _, path := range allSelectors {
   599→		if len(path) < 2 {
   600→			continue
   601→		}
   602→
   603→		key := path[0]   // The "Key" (Level 1)
   604→		value := path[1] // The "Value" (Level 2)
   605→
   606→		exclude := false
   607→
   608→		// Check if the key exists in our exclusion map
   609→		if excludedValues, exists := excludeSelectors[key]; exists {
   610→			// The key exists, now check if the specific value is in the exclusion list
   611→			if slices.Contains(excludedValues, value) {
   612→				exclude = true
   613→			}
   614→		}
   615→
   616→		if !exclude {
   617→			filteredSelectors = append(filteredSelectors, path)
   618→		}
   619→	}
   620→
   621→	return filteredSelectors
   622→}
   623→
   624→// GetPaths returns a list of lists (paths) to the specified depth.
   625→func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
   626→	var results [][]string
   627→
   628→	// Start recursion. Initial path is empty.
   629→	// We treat Root as depth 0.
   630→	ms.root.collectPaths(0, targetDepth, []string{}, &results)
   631→
   632→	return results
   633→}
   634→
   635→// Write all values in `metrics` to the level specified by `selector` for time `ts`.
   636→// Look at `findLevelOrCreate` for how selectors work.
   637→func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
   638→	var ok bool
   639→	for i, metric := range metrics {
   640→		if metric.MetricConfig.Frequency == 0 {
   641→			metric.MetricConfig, ok = m.Metrics[metric.Name]
   642→			if !ok {
   643→				cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
   644→				metric.MetricConfig.Frequency = 0
   645→			}
   646→			metrics[i] = metric
   647→		}
   648→	}
   649→
   650→	return m.WriteToLevel(&m.root, selector, ts, metrics)
   651→}
   652→
   653→func (m *MemoryStore) GetLevel(selector []string) *Level {
   654→	return m.root.findLevelOrCreate(selector, len(m.Metrics))
   655→}
   656→
   657→// WriteToLevel assumes that `minfo` in `metrics` is filled in
   658→func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
   659→	l = l.findLevelOrCreate(selector, len(m.Metrics))
   660→	l.lock.Lock()
   661→	defer l.lock.Unlock()
   662→
   663→	for _, metric := range metrics {
   664→		if metric.MetricConfig.Frequency == 0 {
   665→			continue
   666→		}
   667→
   668→		b := l.metrics[metric.MetricConfig.offset]
   669→		if b == nil {
   670→			// First write to this metric and level
   671→			b = newBuffer(ts, metric.MetricConfig.Frequency)
   672→			l.metrics[metric.MetricConfig.offset] = b
   673→		}
   674→
   675→		nb, err := b.write(ts, metric.Value)
   676→		if err != nil {
   677→			return err
   678→		}
   679→
   680→		// Last write created a new buffer...
   681→		if b != nb {
   682→			l.metrics[metric.MetricConfig.offset] = nb
   683→		}
   684→	}
   685→	return nil
   686→}
   687→
   688→// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
   689→// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
   690→// The second and third return value are the actual from/to for the data. Those can be different from
   691→// the range asked for if no data was available.
   692→func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
   693→	if from > to {
   694→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
   695→	}
   696→
   697→	minfo, ok := m.Metrics[metric]
   698→	if !ok {
   699→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
   700→	}
   701→
   702→	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
   703→
   704→	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
   705→		cdata, cfrom, cto, err := b.read(from, to, data)
   706→		if err != nil {
   707→			return err
   708→		}
   709→
   710→		if n == 0 {
   711→			from, to = cfrom, cto
   712→		} else if from != cfrom || to != cto || len(data) != len(cdata) {
   713→			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
   714→			if missingfront != 0 {
   715→				return ErrDataDoesNotAlignMissingFront
   716→			}
   717→
   718→			newlen := len(cdata) - missingback
   719→			if newlen < 1 {
   720→				return ErrDataDoesNotAlignMissingBack
   721→			}
   722→			cdata = cdata[0:newlen]
   723→			if len(cdata) != len(data) {
   724→				return ErrDataDoesNotAlignDataLenMismatch
   725→			}
   726→
   727→			from, to = cfrom, cto
   728→		}
   729→
   730→		data = cdata
   731→		n += 1
   732→		return nil
   733→	})
   734→
   735→	if err != nil {
   736→		return nil, 0, 0, 0, err
   737→	} else if n == 0 {
   738→		return nil, 0, 0, 0, ErrNoHostOrMetric
   739→	} else if n > 1 {
   740→		if minfo.Aggregation == AvgAggregation {
   741→			normalize := 1. / schema.Float(n)
   742→			for i := 0; i < len(data); i++ {
   743→				data[i] *= normalize
   744→			}
   745→		} else if minfo.Aggregation != SumAggregation {
   746→			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
   747→		}
   748→	}
   749→
   750→	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
   751→	if err != nil {
   752→		return nil, 0, 0, 0, err
   753→	}
   754→
   755→	return data, from, to, resolution, nil
   756→}
   757→
   758→// Free releases all buffers for the selected level and all its children that
   759→// contain only values older than `t`.
   760→func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
   761→	return m.GetLevel(selector).free(t)
   762→}
   763→
   764→// ForceFree unconditionally removes the oldest buffer from each metric chain.
   765→func (m *MemoryStore) ForceFree() (int, error) {
   766→	return m.GetLevel(nil).forceFree()
   767→}
   768→
   769→func (m *MemoryStore) FreeAll() error {
   770→	m.root.lock.Lock()
   771→	defer m.root.lock.Unlock()
   772→
   773→	for k := range m.root.children {
   774→		delete(m.root.children, k)
   775→	}
   776→
   777→	return nil
   778→}
   779→
   780→func (m *MemoryStore) SizeInBytes() int64 {
   781→	return m.root.sizeInBytes()
   782→}
   783→
   784→func (m *MemoryStore) SizeInGB() float64 {
   785→	return float64(m.root.sizeInBytes()) / 1e9
   786→}
   787→
   788→// ListChildren , given a selector, returns a list of all children of the level
   789→// selected.
   790→func (m *MemoryStore) ListChildren(selector []string) []string {
   791→	lvl := &m.root
   792→	for lvl != nil && len(selector) != 0 {
   793→		lvl.lock.RLock()
   794→		next := lvl.children[selector[0]]
   795→		lvl.lock.RUnlock()
   796→		lvl = next
   797→		selector = selector[1:]
   798→	}
   799→
   800→	if lvl == nil {
   801→		return nil
   802→	}
   803→
   804→	lvl.lock.RLock()
   805→	defer lvl.lock.RUnlock()
   806→
   807→	children := make([]string, 0, len(lvl.children))
   808→	for child := range lvl.children {
   809→		children = append(children, child)
   810→	}
   811→
   812→	return children
   813→}
   814→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides an efficient in-memory time-series metric storage system
// with support for hierarchical data organization, checkpointing, and archiving.
//
// The package organizes metrics in a tree structure (cluster → host → component) and
// provides concurrent read/write access to metric data with configurable aggregation strategies.
// Background goroutines handle periodic checkpointing (JSON or Avro format), archiving old data,
// and enforcing retention policies.
//
// Key features:
//   - In-memory metric storage with configurable retention
//   - Hierarchical data organization (selectors)
//   - Concurrent checkpoint/archive workers
//   - Support for sum and average aggregation
//   - NATS integration for metric ingestion
package metricstore

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"runtime"
	"runtime/debug"
	"slices"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/config"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/resampler"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-lib/v2/util"
)

// GlobalState holds the global state for the metric store with thread-safe access.
type GlobalState struct {
	mu                sync.RWMutex
	lastRetentionTime int64
	selectorsExcluded bool
}

var (
	singleton  sync.Once
	msInstance *MemoryStore
	// shutdownFunc stores the context cancellation function created in Init
	// and is called during Shutdown to cancel all background goroutines
	shutdownFunc   context.CancelFunc
	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
	// Create a global instance
	state = &GlobalState{}
)

// NodeProvider provides information about nodes currently in use by running jobs.
//
// This interface allows metricstore to query job information without directly
// depending on the repository package, breaking the import cycle.
//
// Implementations should return nodes that are actively processing jobs started
// before the given timestamp. These nodes will be excluded from retention-based
// garbage collection to prevent data loss for jobs that are still running or
// recently completed.
type NodeProvider interface {
	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
	// that are currently in use by jobs that started before the given timestamp.
	//
	// Parameters:
	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
	//
	// Returns:
	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
	//   - Error if the query fails
	GetUsedNodes(ts int64) (map[string][]string, error)
}

// Metric represents a single metric data point to be written to the store.
type Metric struct {
	Name  string
	Value schema.Float
	// MetricConfig contains frequency and aggregation settings for this metric.
	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
	MetricConfig MetricConfig
}

// MemoryStore is the main in-memory time-series metric storage implementation.
//
// It organizes metrics in a hierarchical tree structure where each level represents
// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
// store multiple metrics as time-series buffers.
//
// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
// All public methods are safe for concurrent use.
type MemoryStore struct {
	Metrics      map[string]MetricConfig
	root         Level
	nodeProvider NodeProvider
}

// Init initializes the metric store from configuration and starts background workers.
//
// This function must be called exactly once before any other metricstore operations.
// It performs the following initialization steps:
//  1. Validates and decodes the metric store configuration
//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
//  3. Loads metric configurations from all registered clusters
//  4. Restores checkpoints within the retention window
//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
//  6. Optionally subscribes to NATS for real-time metric ingestion
//
// Parameters:
//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig)
//   - wg: WaitGroup that will be incremented for each background goroutine started
//
// The function will call cclog.Fatal on critical errors during initialization.
// Use Shutdown() to cleanly stop all background workers started by Init().
//
// Note: Signal handling must be implemented by the caller. Call Shutdown() when
// receiving termination signals to ensure checkpoint data is persisted.
func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
	startupTime := time.Now()

	if rawConfig != nil {
		config.Validate(configSchema, rawConfig)
		dec := json.NewDecoder(bytes.NewReader(rawConfig))
		dec.DisallowUnknownFields()
		if err := dec.Decode(&Keys); err != nil {
			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
		}
	}

	// Set NumWorkers from config or use default
	if Keys.NumWorkers <= 0 {
		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
	}
	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)

	// Pass the config.MetricStoreKeys
	InitMetrics(metrics)

	ms := GetMemoryStore()

	d, err := time.ParseDuration(Keys.RetentionInMemory)
	if err != nil {
		cclog.Fatal(err)
	}

	restoreFrom := startupTime.Add(-d)
	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
	if err != nil {
		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
	} else {
		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
	}

	// Try to use less memory by forcing a GC run here and then
	// lowering the target percentage. The default of 100 means
	// that only once the ratio of new allocations execeds the
	// previously active heap, a GC is triggered.
	// Forcing a GC here will set the "previously active heap"
	// to a minumum.
	// runtime.GC()

	ctx, shutdown := context.WithCancel(context.Background())

	Retention(wg, ctx)
	Checkpointing(wg, ctx)
	CleanUp(wg, ctx)
	WALStaging(wg, ctx)
	MemoryUsageTracker(wg, ctx)

	// Note: Signal handling has been removed from this function.
	// The caller is responsible for handling shutdown signals and calling
	// the shutdown() function when appropriate.
	// Store the shutdown function for later use by Shutdown()
	shutdownFuncMu.Lock()
	shutdownFunc = shutdown
	shutdownFuncMu.Unlock()

	if Keys.Subscriptions != nil {
		wg.Go(func() {
			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
				cclog.Fatal(err)
			}
		})
	}
}

// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
//
// This function must be called before GetMemoryStore() and can only be called once due to
// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
//
// Parameters:
//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
//
// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
//
// After this call, the global msInstance is ready for use via GetMemoryStore().
func InitMetrics(metrics map[string]MetricConfig) {
	singleton.Do(func() {
		offset := 0
		for key, cfg := range metrics {
			if cfg.Frequency == 0 {
				panic("[METRICSTORE]> invalid frequency")
			}

			metrics[key] = MetricConfig{
				Frequency:   cfg.Frequency,
				Aggregation: cfg.Aggregation,
				offset:      offset,
			}
			offset += 1
		}

		msInstance = &MemoryStore{
			root: Level{
				metrics:  make([]*buffer, len(metrics)),
				children: make(map[string]*Level),
			},
			Metrics: metrics,
		}
	})
}

// GetMemoryStore returns the singleton MemoryStore instance.
//
// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
//
// This function is safe for concurrent use after initialization.
func GetMemoryStore() *MemoryStore {
	if msInstance == nil {
		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
	}

	return msInstance
}

func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
	if metric, ok := ms.Metrics[metricName]; ok {
		return metric.Frequency, nil
	}
	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
}

// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
// This must be called during initialization to provide job state information
// for selective buffer retention during Free operations.
// If not set, the Free function will fall back to freeing all buffers.
func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
	ms.nodeProvider = provider
}

// Shutdown performs a graceful shutdown of the metric store.
//
// This function cancels all background goroutines started by Init() and writes
// a final checkpoint to disk before returning. It should be called when the
// application receives a termination signal.
//
// The function will:
//  1. Cancel the context to stop all background workers
//  2. Close the WAL messages channel if using WAL format
//  3. Write a final checkpoint to preserve in-memory data
//  4. Log any errors encountered during shutdown
//
// Note: This function blocks until the final checkpoint is written.
func Shutdown() {
	totalStart := time.Now()

	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}
	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))

	if Keys.Checkpoints.FileFormat == "wal" {
		// Signal producers to stop sending before closing channels,
		// preventing send-on-closed-channel panics from in-flight NATS workers.
		walShuttingDown.Store(true)
		// Brief grace period for in-flight DecodeLine calls to complete.
		time.Sleep(100 * time.Millisecond)

		for _, ch := range walShardChs {
			close(ch)
		}
		drainStart := time.Now()
		WaitForWALStagingDrain()
		cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
	}

	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
	checkpointStart := time.Now()
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		// WAL files are deleted per-host inside ToCheckpointWAL workers.
		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
}

// Retention starts a background goroutine that periodically frees old metric data.
//
// This worker runs at half the retention interval and calls Free() to remove buffers
// older than the configured retention time. It respects the NodeProvider to preserve
// data for nodes with active jobs.
//
// Parameters:
//   - wg: WaitGroup to signal completion when context is cancelled
//   - ctx: Context for cancellation signal
//
// The goroutine exits when ctx is cancelled.
func Retention(wg *sync.WaitGroup, ctx context.Context) {
	ms := GetMemoryStore()

	wg.Go(func() {
		d, err := time.ParseDuration(Keys.RetentionInMemory)
		if err != nil {
			cclog.Fatal(err)
		}
		if d <= 0 {
			return
		}

		tickInterval := d / 2
		if tickInterval <= 0 {
			return
		}
		ticker := time.NewTicker(tickInterval)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				state.mu.Lock()

				t := time.Now().Add(-d)

				state.lastRetentionTime = t.Unix()

				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))

				freed, err := Free(ms, t)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
				} else {
					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
				}

				state.mu.Unlock()

				// Clean up the buffer pool
				bufferPool.Clean(state.lastRetentionTime)
			}
		}
	})
}

// MemoryUsageTracker starts a background goroutine that monitors memory usage.
//
// This worker checks actual process memory usage (via runtime.MemStats) periodically
// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
// performance issues.
//
// The tracker logs both actual memory usage (heap allocated) and metric data size for
// visibility into memory overhead from Go runtime structures and allocations.
//
// Parameters:
//   - wg: WaitGroup to signal completion when context is cancelled
//   - ctx: Context for cancellation signal
//
// The goroutine exits when ctx is cancelled.
func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
	ms := GetMemoryStore()

	wg.Go(func() {
		normalInterval := DefaultMemoryUsageTrackerInterval
		fastInterval := 30 * time.Second

		if normalInterval <= 0 {
			return
		}

		ticker := time.NewTicker(normalInterval)
		defer ticker.Stop()
		currentInterval := normalInterval

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				var mem runtime.MemStats
				runtime.ReadMemStats(&mem)
				actualMemoryGB := float64(mem.Alloc) / 1e9
				metricDataGB := ms.SizeInGB()
				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)

				freedExcluded := 0
				freedEmergency := 0
				var err error

				state.mu.RLock()
				lastRetention := state.lastRetentionTime
				selectorsExcluded := state.selectorsExcluded
				state.mu.RUnlock()

				if lastRetention != 0 && selectorsExcluded {
					freedExcluded, err = ms.Free(nil, lastRetention)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
					}

					if freedExcluded > 0 {
						debug.FreeOSMemory()
						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
					}
				}

				runtime.ReadMemStats(&mem)
				actualMemoryGB = float64(mem.Alloc) / 1e9

				if actualMemoryGB > float64(Keys.MemoryCap) {
					cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)

					// Use progressive time-based Free with increasing threshold
					// instead of ForceFree loop — fewer tree traversals, more effective
					d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
					if parseErr != nil {
						cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
					} else {
						thresholds := []float64{0.75, 0.5, 0.25}
						for _, fraction := range thresholds {
							threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
							freed, freeErr := ms.Free(nil, threshold.Unix())
							if freeErr != nil {
								cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
							}
							freedEmergency += freed

							bufferPool.Clear()
							runtime.GC()
							runtime.ReadMemStats(&mem)
							actualMemoryGB = float64(mem.Alloc) / 1e9

							if actualMemoryGB < float64(Keys.MemoryCap) {
								break
							}
						}
					}

					bufferPool.Clear()
					debug.FreeOSMemory()

					runtime.ReadMemStats(&mem)
					actualMemoryGB = float64(mem.Alloc) / 1e9

					if actualMemoryGB >= float64(Keys.MemoryCap) {
						cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
					} else {
						cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
					}
				}

				// Adaptive ticker: check more frequently when memory is high
				memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
				if memoryRatio > 0.8 && currentInterval != fastInterval {
					ticker.Reset(fastInterval)
					currentInterval = fastInterval
					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
				} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
					ticker.Reset(normalInterval)
					currentInterval = normalInterval
					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
				}
			}
		}
	})
}

// Free removes metric data older than the given time while preserving data for active nodes.
//
// This function implements intelligent retention by consulting the NodeProvider (if configured)
// to determine which nodes are currently in use by running jobs. Data for these nodes is
// preserved even if older than the retention time.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - t: Time threshold - buffers with data older than this will be freed
//
// Returns:
//   - Number of buffers freed
//   - Error if NodeProvider query fails
//
// Behavior:
//   - If no NodeProvider is set: frees all buffers older than t
//   - If NodeProvider returns empty map: frees all buffers older than t
//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
func Free(ms *MemoryStore, t time.Time) (int, error) {
	// If no NodeProvider is configured, free all buffers older than t
	if ms.nodeProvider == nil {
		return ms.Free(nil, t.Unix())
	}

	excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
	if err != nil {
		return 0, err
	}

	switch lenMap := len(excludeSelectors); lenMap {

	// If the length of the map returned by GetUsedNodes() is 0,
	// then use default Free method with nil selector
	case 0:
		state.selectorsExcluded = false
		return ms.Free(nil, t.Unix())

	// Else formulate selectors, exclude those from the map
	// and free the rest of the selectors
	default:
		state.selectorsExcluded = true
		selectors := GetSelectors(ms, excludeSelectors)
		return FreeSelected(ms, selectors, t)
	}
}

// FreeSelected frees buffers for specific selectors while preserving others.
//
// This function is used when we want to retain some specific nodes beyond the retention time.
// It iterates through the provided selectors and frees their associated buffers.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
//   - t: Time threshold for freeing buffers
//
// Returns the total number of buffers freed and any error encountered.
func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
	freed := 0

	for _, selector := range selectors {

		freedBuffers, err := ms.Free(selector, t.Unix())
		if err != nil {
			cclog.Errorf("error while freeing selected buffers: %#v", err)
		}
		freed += freedBuffers

	}

	return freed, nil
}

// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
//
// This function generates a list of selectors whose buffers should be freed by excluding
// selectors that correspond to nodes currently in use by running jobs.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
//
// Returns a list of selectors ([]string paths) that can be safely freed.
//
// Example:
//
//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
//	and excludeSelectors contains {"emmy": ["node001"]},
//	then only [["emmy", "node002"]] is returned.
func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
	allSelectors := ms.GetPaths(2)

	filteredSelectors := make([][]string, 0, len(allSelectors))

	for _, path := range allSelectors {
		if len(path) < 2 {
			continue
		}

		key := path[0]   // The "Key" (Level 1)
		value := path[1] // The "Value" (Level 2)

		exclude := false

		// Check if the key exists in our exclusion map
		if excludedValues, exists := excludeSelectors[key]; exists {
			// The key exists, now check if the specific value is in the exclusion list
			if slices.Contains(excludedValues, value) {
				exclude = true
			}
		}

		if !exclude {
			filteredSelectors = append(filteredSelectors, path)
		}
	}

	return filteredSelectors
}

// GetPaths returns a list of lists (paths) to the specified depth.
func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
	var results [][]string

	// Start recursion. Initial path is empty.
	// We treat Root as depth 0.
	ms.root.collectPaths(0, targetDepth, []string{}, &results)

	return results
}

// Write all values in `metrics` to the level specified by `selector` for time `ts`.
// Look at `findLevelOrCreate` for how selectors work.
func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
	var ok bool
	for i, metric := range metrics {
		if metric.MetricConfig.Frequency == 0 {
			metric.MetricConfig, ok = m.Metrics[metric.Name]
			if !ok {
				cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
				metric.MetricConfig.Frequency = 0
			}
			metrics[i] = metric
		}
	}

	return m.WriteToLevel(&m.root, selector, ts, metrics)
}

func (m *MemoryStore) GetLevel(selector []string) *Level {
	return m.root.findLevelOrCreate(selector, len(m.Metrics))
}

// WriteToLevel assumes that `minfo` in `metrics` is filled in
func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
	l = l.findLevelOrCreate(selector, len(m.Metrics))
	l.lock.Lock()
	defer l.lock.Unlock()

	for _, metric := range metrics {
		if metric.MetricConfig.Frequency == 0 {
			continue
		}

		b := l.metrics[metric.MetricConfig.offset]
		if b == nil {
			// First write to this metric and level
			b = newBuffer(ts, metric.MetricConfig.Frequency)
			l.metrics[metric.MetricConfig.offset] = b
		}

		nb, err := b.write(ts, metric.Value)
		if err != nil {
			return err
		}

		// Last write created a new buffer...
		if b != nb {
			l.metrics[metric.MetricConfig.offset] = nb
		}
	}
	return nil
}

// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
// The second and third return value are the actual from/to for the data. Those can be different from
// the range asked for if no data was available.
func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
	if from > to {
		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
	}

	minfo, ok := m.Metrics[metric]
	if !ok {
		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
	}

	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)

	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
		cdata, cfrom, cto, err := b.read(from, to, data)
		if err != nil {
			return err
		}

		if n == 0 {
			from, to = cfrom, cto
		} else if from != cfrom || to != cto || len(data) != len(cdata) {
			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
			if missingfront != 0 {
				return ErrDataDoesNotAlignMissingFront
			}

			newlen := len(cdata) - missingback
			if newlen < 1 {
				return ErrDataDoesNotAlignMissingBack
			}
			cdata = cdata[0:newlen]
			if len(cdata) != len(data) {
				return ErrDataDoesNotAlignDataLenMismatch
			}

			from, to = cfrom, cto
		}

		data = cdata
		n += 1
		return nil
	})

	if err != nil {
		return nil, 0, 0, 0, err
	} else if n == 0 {
		return nil, 0, 0, 0, ErrNoHostOrMetric
	} else if n > 1 {
		if minfo.Aggregation == AvgAggregation {
			normalize := 1. / schema.Float(n)
			for i := 0; i < len(data); i++ {
				data[i] *= normalize
			}
		} else if minfo.Aggregation != SumAggregation {
			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
		}
	}

	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
	if err != nil {
		return nil, 0, 0, 0, err
	}

	return data, from, to, resolution, nil
}

// Free releases all buffers for the selected level and all its children that
// contain only values older than `t`.
func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
	return m.GetLevel(selector).free(t)
}

// ForceFree unconditionally removes the oldest buffer from each metric chain.
func (m *MemoryStore) ForceFree() (int, error) {
	return m.GetLevel(nil).forceFree()
}

func (m *MemoryStore) FreeAll() error {
	m.root.lock.Lock()
	defer m.root.lock.Unlock()

	for k := range m.root.children {
		delete(m.root.children, k)
	}

	return nil
}

func (m *MemoryStore) SizeInBytes() int64 {
	return m.root.sizeInBytes()
}

func (m *MemoryStore) SizeInGB() float64 {
	return float64(m.root.sizeInBytes()) / 1e9
}

// ListChildren , given a selector, returns a list of all children of the level
// selected.
func (m *MemoryStore) ListChildren(selector []string) []string {
	lvl := &m.root
	for lvl != nil && len(selector) != 0 {
		lvl.lock.RLock()
		next := lvl.children[selector[0]]
		lvl.lock.RUnlock()
		lvl = next
		selector = selector[1:]
	}

	if lvl == nil {
		return nil
	}

	lvl.lock.RLock()
	defer lvl.lock.RUnlock()

	children := make([]string, 0, len(lvl.children))
	for child := range lvl.children {
		children = append(children, child)
	}

	return children
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file contains the API types and data fetching logic for querying metric data
     7→// from the in-memory metric store. It provides structures for building complex queries
     8→// with support for aggregation, scaling, padding, and statistics computation.
     9→package metricstore
    10→
    11→import (
    12→	"errors"
    13→	"fmt"
    14→	"math"
    15→
    16→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    17→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    18→	"github.com/ClusterCockpit/cc-lib/v2/util"
    19→)
    20→
    21→var (
    22→	// ErrNoHostOrMetric is returned when the metric store does not find the host or the metric
    23→	ErrNoHostOrMetric error = errors.New("[METRICSTORE]> metric or host not found")
    24→	// ErrInvalidTimeRange is returned when a query has 'from' >= 'to'
    25→	ErrInvalidTimeRange = errors.New("[METRICSTORE]> invalid time range: 'from' must be before 'to'")
    26→	// ErrEmptyCluster is returned when a query with ForAllNodes has no cluster specified
    27→	ErrEmptyCluster = errors.New("[METRICSTORE]> cluster name cannot be empty")
    28→)
    29→
    30→// APIMetricData represents the response data for a single metric query.
    31→//
    32→// It contains both the time-series data points and computed statistics (avg, min, max).
    33→// If an error occurred during data retrieval, the Error field will be set and other
    34→// fields may be incomplete.
    35→type APIMetricData struct {
    36→	Error      *string           `json:"error,omitempty"`
    37→	Data       schema.FloatArray `json:"data,omitempty"`
    38→	From       int64             `json:"from"`
    39→	To         int64             `json:"to"`
    40→	Resolution int64             `json:"resolution"`
    41→	Avg        schema.Float      `json:"avg"`
    42→	Min        schema.Float      `json:"min"`
    43→	Max        schema.Float      `json:"max"`
    44→}
    45→
    46→// APIQueryRequest represents a batch query request for metric data.
    47→//
    48→// It supports two modes of operation:
    49→//  1. Explicit queries via the Queries field
    50→//  2. Automatic query generation via ForAllNodes (queries all specified metrics for all nodes in the cluster)
    51→//
    52→// The request can be customized with flags to include/exclude statistics, raw data, and padding.
    53→type APIQueryRequest struct {
    54→	Cluster     string     `json:"cluster"`
    55→	Queries     []APIQuery `json:"queries"`
    56→	ForAllNodes []string   `json:"for-all-nodes"`
    57→	From        int64      `json:"from"`
    58→	To          int64      `json:"to"`
    59→	WithStats   bool       `json:"with-stats"`
    60→	WithData    bool       `json:"with-data"`
    61→	WithPadding bool       `json:"with-padding"`
    62→}
    63→
    64→// APIQueryResponse represents the response to an APIQueryRequest.
    65→//
    66→// Results is a 2D array where each outer element corresponds to a query,
    67→// and each inner element corresponds to a selector within that query
    68→// (e.g., multiple CPUs or cores).
    69→type APIQueryResponse struct {
    70→	Queries []APIQuery        `json:"queries,omitempty"`
    71→	Results [][]APIMetricData `json:"results"`
    72→}
    73→
    74→// APIQuery represents a single metric query with optional hierarchical selectors.
    75→//
    76→// The hierarchical selection works as follows:
    77→//   - Hostname: The node to query
    78→//   - Type + TypeIds: First level of hierarchy (e.g., "cpu" + ["0", "1", "2"])
    79→//   - SubType + SubTypeIds: Second level of hierarchy (e.g., "core" + ["0", "1"])
    80→//
    81→// If Aggregate is true, data from multiple type/subtype IDs will be aggregated according
    82→// to the metric's aggregation strategy. Otherwise, separate results are returned for each combination.
    83→type APIQuery struct {
    84→	Type        *string      `json:"type,omitempty"`
    85→	SubType     *string      `json:"subtype,omitempty"`
    86→	Metric      string       `json:"metric"`
    87→	Hostname    string       `json:"host"`
    88→	Resolution  int64        `json:"resolution"`
    89→	TypeIds     []string     `json:"type-ids,omitempty"`
    90→	SubTypeIds  []string     `json:"subtype-ids,omitempty"`
    91→	ScaleFactor schema.Float `json:"scale-by,omitempty"`
    92→	Aggregate   bool         `json:"aggreg"`
    93→}
    94→
    95→// AddStats computes and populates the Avg, Min, and Max fields from the Data array.
    96→//
    97→// NaN values in the data are ignored during computation. If all values are NaN,
    98→// the statistics fields will be set to NaN.
    99→//
   100→// TODO: Optimize this, just like the stats endpoint!
   101→func (data *APIMetricData) AddStats() {
   102→	n := 0
   103→	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
   104→	for _, x := range data.Data {
   105→		if x.IsNaN() {
   106→			continue
   107→		}
   108→
   109→		n += 1
   110→		sum += float64(x)
   111→		min = math.Min(min, float64(x))
   112→		max = math.Max(max, float64(x))
   113→	}
   114→
   115→	if n > 0 {
   116→		avg := sum / float64(n)
   117→		data.Avg = schema.Float(avg)
   118→		data.Min = schema.Float(min)
   119→		data.Max = schema.Float(max)
   120→	} else {
   121→		data.Avg, data.Min, data.Max = schema.NaN, schema.NaN, schema.NaN
   122→	}
   123→}
   124→
   125→// ScaleBy multiplies all data points and statistics by the given factor.
   126→//
   127→// This is commonly used for unit conversion (e.g., bytes to gigabytes).
   128→// Scaling by 0 or 1 is a no-op for performance reasons.
   129→func (data *APIMetricData) ScaleBy(f schema.Float) {
   130→	if f == 1 {
   131→		return
   132→	}
   133→
   134→	data.Avg *= f
   135→	data.Min *= f
   136→	data.Max *= f
   137→	for i := 0; i < len(data.Data); i++ {
   138→		data.Data[i] *= f
   139→	}
   140→}
   141→
   142→// PadDataWithNull pads the beginning of the data array with NaN values if needed.
   143→//
   144→// This ensures that the data aligns with the requested 'from' timestamp, even if
   145→// the metric store doesn't have data for the earliest time points. This is useful
   146→// for maintaining consistent array indexing across multiple queries.
   147→//
   148→// Parameters:
   149→//   - ms: MemoryStore instance to lookup metric configuration
   150→//   - from: The requested start timestamp
   151→//   - to: The requested end timestamp (unused but kept for API consistency)
   152→//   - metric: The metric name to lookup frequency information
   153→func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metric string) {
   154→	minfo, ok := ms.Metrics[metric]
   155→	if !ok {
   156→		return
   157→	}
   158→
   159→	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
   160→		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
   161→		ndata := make([]schema.Float, 0, padfront+len(data.Data))
   162→		for range padfront {
   163→			ndata = append(ndata, schema.NaN)
   164→		}
   165→		for j := 0; j < len(data.Data); j++ {
   166→			ndata = append(ndata, data.Data[j])
   167→		}
   168→		data.Data = ndata
   169→	}
   170→}
   171→
   172→// FetchData executes a batch metric query request and returns the results.
   173→//
   174→// This is the primary API for retrieving metric data from the memory store. It supports:
   175→//   - Individual queries via req.Queries
   176→//   - Batch queries for all nodes via req.ForAllNodes
   177→//   - Hierarchical selector construction (cluster → host → type → subtype)
   178→//   - Optional statistics computation (avg, min, max)
   179→//   - Optional data scaling
   180→//   - Optional data padding with NaN values
   181→//
   182→// The function constructs selectors based on the query parameters and calls MemoryStore.Read()
   183→// for each selector. If a query specifies Aggregate=false with multiple type/subtype IDs,
   184→// separate results are returned for each combination.
   185→//
   186→// Parameters:
   187→//   - req: The query request containing queries, time range, and options
   188→//
   189→// Returns:
   190→//   - APIQueryResponse containing results for each query, or error if validation fails
   191→//
   192→// Errors:
   193→//   - ErrInvalidTimeRange if req.From > req.To
   194→//   - ErrEmptyCluster if req.ForAllNodes is used without specifying a cluster
   195→//   - Error if MemoryStore is not initialized
   196→//   - Individual query errors are stored in APIMetricData.Error field
   197→func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
   198→	if req.From > req.To {
   199→		return nil, ErrInvalidTimeRange
   200→	}
   201→	if req.Cluster == "" && req.ForAllNodes != nil {
   202→		return nil, ErrEmptyCluster
   203→	}
   204→
   205→	req.WithData = true
   206→	ms := GetMemoryStore()
   207→	if ms == nil {
   208→		return nil, fmt.Errorf("[METRICSTORE]> memorystore not initialized")
   209→	}
   210→
   211→	response := APIQueryResponse{
   212→		Results: make([][]APIMetricData, 0, len(req.Queries)),
   213→	}
   214→	if req.ForAllNodes != nil {
   215→		nodes := ms.ListChildren([]string{req.Cluster})
   216→		for _, node := range nodes {
   217→			for _, metric := range req.ForAllNodes {
   218→				q := APIQuery{
   219→					Metric:   metric,
   220→					Hostname: node,
   221→				}
   222→				req.Queries = append(req.Queries, q)
   223→				response.Queries = append(response.Queries, q)
   224→			}
   225→		}
   226→	}
   227→
   228→	for _, query := range req.Queries {
   229→		sels := make([]util.Selector, 0, 1)
   230→		if query.Aggregate || query.Type == nil {
   231→			sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}}
   232→			if query.Type != nil {
   233→				if len(query.TypeIds) == 1 {
   234→					sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]})
   235→				} else {
   236→					ids := make([]string, len(query.TypeIds))
   237→					for i, id := range query.TypeIds {
   238→						ids[i] = *query.Type + id
   239→					}
   240→					sel = append(sel, util.SelectorElement{Group: ids})
   241→				}
   242→
   243→				if query.SubType != nil {
   244→					if len(query.SubTypeIds) == 1 {
   245→						sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
   246→					} else {
   247→						ids := make([]string, len(query.SubTypeIds))
   248→						for i, id := range query.SubTypeIds {
   249→							ids[i] = *query.SubType + id
   250→						}
   251→						sel = append(sel, util.SelectorElement{Group: ids})
   252→					}
   253→				}
   254→			}
   255→			sels = append(sels, sel)
   256→		} else {
   257→			for _, typeID := range query.TypeIds {
   258→				if query.SubType != nil {
   259→					for _, subTypeID := range query.SubTypeIds {
   260→						sels = append(sels, util.Selector{
   261→							{String: req.Cluster},
   262→							{String: query.Hostname},
   263→							{String: *query.Type + typeID},
   264→							{String: *query.SubType + subTypeID},
   265→						})
   266→					}
   267→				} else {
   268→					sels = append(sels, util.Selector{
   269→						{String: req.Cluster},
   270→						{String: query.Hostname},
   271→						{String: *query.Type + typeID},
   272→					})
   273→				}
   274→			}
   275→		}
   276→
   277→		var err error
   278→		res := make([]APIMetricData, 0, len(sels))
   279→		for _, sel := range sels {
   280→			data := APIMetricData{}
   281→
   282→			data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
   283→			if err != nil {
   284→				// Skip Error If Just Missing Host or Metric, Continue
   285→				// Empty Return For Metric Handled Gracefully By Frontend
   286→				if err != ErrNoHostOrMetric {
   287→					msg := err.Error()
   288→					data.Error = &msg
   289→					res = append(res, data)
   290→				} else {
   291→					cclog.Debugf("failed to fetch '%s' from host '%s' (cluster: %s): %s", query.Metric, query.Hostname, req.Cluster, err.Error())
   292→				}
   293→				continue
   294→			}
   295→
   296→			if req.WithStats {
   297→				data.AddStats()
   298→			}
   299→			if query.ScaleFactor != 0 {
   300→				data.ScaleBy(query.ScaleFactor)
   301→			}
   302→			if req.WithPadding {
   303→				data.PadDataWithNull(ms, req.From, req.To, query.Metric)
   304→			}
   305→			if !req.WithData {
   306→				data.Data = nil
   307→			}
   308→			res = append(res, data)
   309→		}
   310→		response.Results = append(response.Results, res)
   311→	}
   312→
   313→	return &response, nil
   314→}
   315→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file contains the API types and data fetching logic for querying metric data
// from the in-memory metric store. It provides structures for building complex queries
// with support for aggregation, scaling, padding, and statistics computation.
package metricstore

import (
	"errors"
	"fmt"
	"math"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-lib/v2/util"
)

var (
	// ErrNoHostOrMetric is returned when the metric store does not find the host or the metric
	ErrNoHostOrMetric error = errors.New("[METRICSTORE]> metric or host not found")
	// ErrInvalidTimeRange is returned when a query has 'from' >= 'to'
	ErrInvalidTimeRange = errors.New("[METRICSTORE]> invalid time range: 'from' must be before 'to'")
	// ErrEmptyCluster is returned when a query with ForAllNodes has no cluster specified
	ErrEmptyCluster = errors.New("[METRICSTORE]> cluster name cannot be empty")
)

// APIMetricData represents the response data for a single metric query.
//
// It contains both the time-series data points and computed statistics (avg, min, max).
// If an error occurred during data retrieval, the Error field will be set and other
// fields may be incomplete.
type APIMetricData struct {
	Error      *string           `json:"error,omitempty"`
	Data       schema.FloatArray `json:"data,omitempty"`
	From       int64             `json:"from"`
	To         int64             `json:"to"`
	Resolution int64             `json:"resolution"`
	Avg        schema.Float      `json:"avg"`
	Min        schema.Float      `json:"min"`
	Max        schema.Float      `json:"max"`
}

// APIQueryRequest represents a batch query request for metric data.
//
// It supports two modes of operation:
//  1. Explicit queries via the Queries field
//  2. Automatic query generation via ForAllNodes (queries all specified metrics for all nodes in the cluster)
//
// The request can be customized with flags to include/exclude statistics, raw data, and padding.
type APIQueryRequest struct {
	Cluster     string     `json:"cluster"`
	Queries     []APIQuery `json:"queries"`
	ForAllNodes []string   `json:"for-all-nodes"`
	From        int64      `json:"from"`
	To          int64      `json:"to"`
	WithStats   bool       `json:"with-stats"`
	WithData    bool       `json:"with-data"`
	WithPadding bool       `json:"with-padding"`
}

// APIQueryResponse represents the response to an APIQueryRequest.
//
// Results is a 2D array where each outer element corresponds to a query,
// and each inner element corresponds to a selector within that query
// (e.g., multiple CPUs or cores).
type APIQueryResponse struct {
	Queries []APIQuery        `json:"queries,omitempty"`
	Results [][]APIMetricData `json:"results"`
}

// APIQuery represents a single metric query with optional hierarchical selectors.
//
// The hierarchical selection works as follows:
//   - Hostname: The node to query
//   - Type + TypeIds: First level of hierarchy (e.g., "cpu" + ["0", "1", "2"])
//   - SubType + SubTypeIds: Second level of hierarchy (e.g., "core" + ["0", "1"])
//
// If Aggregate is true, data from multiple type/subtype IDs will be aggregated according
// to the metric's aggregation strategy. Otherwise, separate results are returned for each combination.
type APIQuery struct {
	Type        *string      `json:"type,omitempty"`
	SubType     *string      `json:"subtype,omitempty"`
	Metric      string       `json:"metric"`
	Hostname    string       `json:"host"`
	Resolution  int64        `json:"resolution"`
	TypeIds     []string     `json:"type-ids,omitempty"`
	SubTypeIds  []string     `json:"subtype-ids,omitempty"`
	ScaleFactor schema.Float `json:"scale-by,omitempty"`
	Aggregate   bool         `json:"aggreg"`
}

// AddStats computes and populates the Avg, Min, and Max fields from the Data array.
//
// NaN values in the data are ignored during computation. If all values are NaN,
// the statistics fields will be set to NaN.
//
// TODO: Optimize this, just like the stats endpoint!
func (data *APIMetricData) AddStats() {
	n := 0
	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
	for _, x := range data.Data {
		if x.IsNaN() {
			continue
		}

		n += 1
		sum += float64(x)
		min = math.Min(min, float64(x))
		max = math.Max(max, float64(x))
	}

	if n > 0 {
		avg := sum / float64(n)
		data.Avg = schema.Float(avg)
		data.Min = schema.Float(min)
		data.Max = schema.Float(max)
	} else {
		data.Avg, data.Min, data.Max = schema.NaN, schema.NaN, schema.NaN
	}
}

// ScaleBy multiplies all data points and statistics by the given factor.
//
// This is commonly used for unit conversion (e.g., bytes to gigabytes).
// Scaling by 0 or 1 is a no-op for performance reasons.
func (data *APIMetricData) ScaleBy(f schema.Float) {
	if f == 1 {
		return
	}

	data.Avg *= f
	data.Min *= f
	data.Max *= f
	for i := 0; i < len(data.Data); i++ {
		data.Data[i] *= f
	}
}

// PadDataWithNull pads the beginning of the data array with NaN values if needed.
//
// This ensures that the data aligns with the requested 'from' timestamp, even if
// the metric store doesn't have data for the earliest time points. This is useful
// for maintaining consistent array indexing across multiple queries.
//
// Parameters:
//   - ms: MemoryStore instance to lookup metric configuration
//   - from: The requested start timestamp
//   - to: The requested end timestamp (unused but kept for API consistency)
//   - metric: The metric name to lookup frequency information
func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metric string) {
	minfo, ok := ms.Metrics[metric]
	if !ok {
		return
	}

	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
		ndata := make([]schema.Float, 0, padfront+len(data.Data))
		for range padfront {
			ndata = append(ndata, schema.NaN)
		}
		for j := 0; j < len(data.Data); j++ {
			ndata = append(ndata, data.Data[j])
		}
		data.Data = ndata
	}
}

// FetchData executes a batch metric query request and returns the results.
//
// This is the primary API for retrieving metric data from the memory store. It supports:
//   - Individual queries via req.Queries
//   - Batch queries for all nodes via req.ForAllNodes
//   - Hierarchical selector construction (cluster → host → type → subtype)
//   - Optional statistics computation (avg, min, max)
//   - Optional data scaling
//   - Optional data padding with NaN values
//
// The function constructs selectors based on the query parameters and calls MemoryStore.Read()
// for each selector. If a query specifies Aggregate=false with multiple type/subtype IDs,
// separate results are returned for each combination.
//
// Parameters:
//   - req: The query request containing queries, time range, and options
//
// Returns:
//   - APIQueryResponse containing results for each query, or error if validation fails
//
// Errors:
//   - ErrInvalidTimeRange if req.From > req.To
//   - ErrEmptyCluster if req.ForAllNodes is used without specifying a cluster
//   - Error if MemoryStore is not initialized
//   - Individual query errors are stored in APIMetricData.Error field
func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
	if req.From > req.To {
		return nil, ErrInvalidTimeRange
	}
	if req.Cluster == "" && req.ForAllNodes != nil {
		return nil, ErrEmptyCluster
	}

	req.WithData = true
	ms := GetMemoryStore()
	if ms == nil {
		return nil, fmt.Errorf("[METRICSTORE]> memorystore not initialized")
	}

	response := APIQueryResponse{
		Results: make([][]APIMetricData, 0, len(req.Queries)),
	}
	if req.ForAllNodes != nil {
		nodes := ms.ListChildren([]string{req.Cluster})
		for _, node := range nodes {
			for _, metric := range req.ForAllNodes {
				q := APIQuery{
					Metric:   metric,
					Hostname: node,
				}
				req.Queries = append(req.Queries, q)
				response.Queries = append(response.Queries, q)
			}
		}
	}

	for _, query := range req.Queries {
		sels := make([]util.Selector, 0, 1)
		if query.Aggregate || query.Type == nil {
			sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}}
			if query.Type != nil {
				if len(query.TypeIds) == 1 {
					sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]})
				} else {
					ids := make([]string, len(query.TypeIds))
					for i, id := range query.TypeIds {
						ids[i] = *query.Type + id
					}
					sel = append(sel, util.SelectorElement{Group: ids})
				}

				if query.SubType != nil {
					if len(query.SubTypeIds) == 1 {
						sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
					} else {
						ids := make([]string, len(query.SubTypeIds))
						for i, id := range query.SubTypeIds {
							ids[i] = *query.SubType + id
						}
						sel = append(sel, util.SelectorElement{Group: ids})
					}
				}
			}
			sels = append(sels, sel)
		} else {
			for _, typeID := range query.TypeIds {
				if query.SubType != nil {
					for _, subTypeID := range query.SubTypeIds {
						sels = append(sels, util.Selector{
							{String: req.Cluster},
							{String: query.Hostname},
							{String: *query.Type + typeID},
							{String: *query.SubType + subTypeID},
						})
					}
				} else {
					sels = append(sels, util.Selector{
						{String: req.Cluster},
						{String: query.Hostname},
						{String: *query.Type + typeID},
					})
				}
			}
		}

		var err error
		res := make([]APIMetricData, 0, len(sels))
		for _, sel := range sels {
			data := APIMetricData{}

			data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
			if err != nil {
				// Skip Error If Just Missing Host or Metric, Continue
				// Empty Return For Metric Handled Gracefully By Frontend
				if err != ErrNoHostOrMetric {
					msg := err.Error()
					data.Error = &msg
					res = append(res, data)
				} else {
					cclog.Debugf("failed to fetch '%s' from host '%s' (cluster: %s): %s", query.Metric, query.Hostname, req.Cluster, err.Error())
				}
				continue
			}

			if req.WithStats {
				data.AddStats()
			}
			if query.ScaleFactor != 0 {
				data.ScaleBy(query.ScaleFactor)
			}
			if req.WithPadding {
				data.PadDataWithNull(ms, req.From, req.To, query.Metric)
			}
			if !req.WithData {
				data.Data = nil
			}
			res = append(res, data)
		}
		response.Results = append(response.Results, res)
	}

	return &response, nil
}