     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements ingestion of InfluxDB line-protocol metric data received
     7→// over NATS. Each line encodes one metric sample with the following structure:
     8→//
     9→//	<measurement>[,cluster=<c>][,hostname=<h>][,type=<t>][,type-id=<id>][,subtype=<s>][,stype-id=<id>] value=<v> [<timestamp>]
    10→//
    11→// The measurement name identifies the metric (e.g. "cpu_load"). Tags provide
    12→// routing information (cluster, host) and optional sub-device selectors (type,
    13→// subtype). Only one field is expected per line: "value".
    14→//
    15→// After decoding, each sample is:
    16→//  1. Written to the in-memory store via ms.WriteToLevel.
    17→//  2. If the checkpoint format is "wal", also forwarded to the WAL staging
    18→//     goroutine via the WALMessages channel for durable write-ahead logging.
    19→package metricstore
    20→
    21→import (
    22→	"bytes"
    23→	"context"
    24→	"fmt"
    25→	"sync"
    26→	"sync/atomic"
    27→	"time"
    28→
    29→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    30→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    31→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    32→	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
    33→)
    34→
    35→// ReceiveNats subscribes to all configured NATS subjects and feeds incoming
    36→// line-protocol messages into the MemoryStore.
    37→//
    38→// When workers > 1 a pool of goroutines drains a shared channel so that
    39→// multiple messages can be decoded in parallel. With workers == 1 the NATS
    40→// callback decodes inline (no channel overhead, lower latency).
    41→//
    42→// The function blocks until ctx is cancelled and all worker goroutines have
    43→// finished. It returns nil when the NATS client is not configured; callers
    44→// should treat that as a no-op rather than an error.
    45→func ReceiveNats(ms *MemoryStore,
    46→	workers int,
    47→	ctx context.Context,
    48→) error {
    49→	nc := nats.GetClient()
    50→
    51→	if nc == nil {
    52→		cclog.Warn("NATS client not initialized")
    53→		return nil
    54→	}
    55→
    56→	var wg sync.WaitGroup
    57→	msgs := make(chan []byte, max(workers*256, 8192))
    58→
    59→	for _, sc := range *Keys.Subscriptions {
    60→		clusterTag := sc.ClusterTag
    61→		if workers > 1 {
    62→			wg.Add(workers)
    63→
    64→			for range workers {
    65→				go func() {
    66→					defer wg.Done()
    67→					for m := range msgs {
    68→						dec := lineprotocol.NewDecoderWithBytes(m)
    69→						if err := DecodeLine(dec, ms, clusterTag); err != nil {
    70→							cclog.Errorf("error: %s", err.Error())
    71→						}
    72→					}
    73→				}()
    74→			}
    75→
    76→			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
    77→				select {
    78→				case msgs <- data:
    79→				case <-ctx.Done():
    80→				}
    81→			})
    82→		} else {
    83→			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
    84→				dec := lineprotocol.NewDecoderWithBytes(data)
    85→				if err := DecodeLine(dec, ms, clusterTag); err != nil {
    86→					cclog.Errorf("error: %s", err.Error())
    87→				}
    88→			})
    89→		}
    90→		cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo)
    91→	}
    92→
    93→	go func() {
    94→		<-ctx.Done()
    95→		close(msgs)
    96→	}()
    97→
    98→	wg.Wait()
    99→
   100→	return nil
   101→}
   102→
   103→// reorder prepends prefix to buf in-place when buf has enough spare capacity,
   104→// avoiding an allocation. Falls back to a regular append otherwise.
   105→//
   106→// It is used to assemble the "type<type-id>" and "subtype<stype-id>" selector
   107→// strings when the type tag arrives before the type-id tag in the line, so the
   108→// two byte slices need to be concatenated in tag-declaration order regardless
   109→// of wire order.
   110→func reorder(buf, prefix []byte) []byte {
   111→	n := len(prefix)
   112→	m := len(buf)
   113→	if cap(buf) < m+n {
   114→		return append(prefix[:n:n], buf...)
   115→	} else {
   116→		buf = buf[:n+m]
   117→		for i := m - 1; i >= 0; i-- {
   118→			buf[i+n] = buf[i]
   119→		}
   120→		for i := range n {
   121→			buf[i] = prefix[i]
   122→		}
   123→		return buf
   124→	}
   125→}
   126→
   127→// walDropped counts WAL messages dropped due to a full channel.
   128→// Logged periodically; not critical since binary snapshots capture the data.
   129→var walDropped atomic.Int64
   130→
   131→// decodeState holds the per-call scratch buffers used by DecodeLine.
   132→// Instances are recycled via decodeStatePool to avoid repeated allocations
   133→// during high-throughput ingestion.
   134→type decodeState struct {
   135→	// metricBuf holds a copy of the current measurement name (line-protocol
   136→	// measurement field). Copied because dec.Measurement() returns a slice
   137→	// that is invalidated by the next decoder call.
   138→	metricBuf []byte
   139→
   140→	// selector is the sub-device path passed to WriteToLevel and WALMessage
   141→	// (e.g. ["socket0"] or ["socket0", "memctrl1"]). Reused across lines.
   142→	selector []string
   143→
   144→	// typeBuf accumulates the concatenated "type"+"type-id" tag value for the
   145→	// current line. Reset at the start of each line's tag-decode loop.
   146→	typeBuf []byte
   147→
   148→	// subTypeBuf accumulates the concatenated "subtype"+"stype-id" tag value.
   149→	// Reset at the start of each line's tag-decode loop.
   150→	subTypeBuf []byte
   151→
   152→	// prevTypeBytes / prevTypeStr cache the last seen typeBuf content and its
   153→	// string conversion. Because consecutive lines in a batch typically address
   154→	// the same sub-device, the cache hit rate is very high and avoids
   155→	// repeated []byte→string allocations.
   156→	prevTypeBytes []byte
   157→	prevTypeStr   string
   158→
   159→	// prevSubTypeBytes / prevSubTypeStr are the same cache for the subtype.
   160→	prevSubTypeBytes []byte
   161→	prevSubTypeStr   string
   162→}
   163→
   164→// decodeStatePool recycles decodeState values across DecodeLine calls to
   165→// reduce GC pressure during sustained metric ingestion.
   166→var decodeStatePool = sync.Pool{
   167→	New: func() any {
   168→		return &decodeState{
   169→			metricBuf:  make([]byte, 0, 16),
   170→			selector:   make([]string, 0, 4),
   171→			typeBuf:    make([]byte, 0, 16),
   172→			subTypeBuf: make([]byte, 0, 16),
   173→		}
   174→	},
   175→}
   176→
   177→// DecodeLine reads all lines from dec (InfluxDB line-protocol) and writes each
   178→// decoded metric sample into ms.
   179→//
   180→// clusterDefault is used as the cluster name for lines that do not carry a
   181→// "cluster" tag. Callers typically supply the ClusterTag value from the NATS
   182→// subscription configuration.
   183→//
   184→// Performance notes:
   185→//   - A decodeState is obtained from decodeStatePool to reuse scratch buffers.
   186→//   - The Level pointer (host-level node in the metric tree) is cached across
   187→//     consecutive lines that share the same cluster+host pair to avoid
   188→//     repeated lock acquisitions on the root and cluster levels.
   189→//   - []byte→string conversions for type/subtype selectors are cached via
   190→//     prevType*/prevSubType* fields because batches typically repeat the same
   191→//     sub-device identifiers.
   192→//   - Timestamp parsing tries Second precision first; if that fails it retries
   193→//     Millisecond, Microsecond, and Nanosecond in turn. A missing timestamp
   194→//     falls back to time.Now().
   195→//
   196→// When the checkpoint format is "wal" each successfully decoded sample is also
   197→// sent to WALMessages so the WAL staging goroutine can persist it durably
   198→// before the next binary snapshot.
   199→func DecodeLine(dec *lineprotocol.Decoder,
   200→	ms *MemoryStore,
   201→	clusterDefault string,
   202→) error {
   203→	// Reduce allocations in loop:
   204→	t := time.Now()
   205→	metric := Metric{}
   206→	st := decodeStatePool.Get().(*decodeState)
   207→	defer decodeStatePool.Put(st)
   208→
   209→	// Optimize for the case where all lines in a "batch" are about the same
   210→	// cluster and host. By using `WriteToLevel` (level = host), we do not need
   211→	// to take the root- and cluster-level lock as often.
   212→	var lvl *Level = nil
   213→	prevCluster, prevHost := "", ""
   214→
   215→	var ok bool
   216→	for dec.Next() {
   217→		rawmeasurement, err := dec.Measurement()
   218→		if err != nil {
   219→			return err
   220→		}
   221→
   222→		// Needs to be copied because another call to dec.* would
   223→		// invalidate the returned slice.
   224→		st.metricBuf = append(st.metricBuf[:0], rawmeasurement...)
   225→
   226→		// The go compiler optimizes map[string(byteslice)] lookups:
   227→		metric.MetricConfig, ok = ms.Metrics[string(rawmeasurement)]
   228→		if !ok {
   229→			continue
   230→		}
   231→
   232→		st.typeBuf, st.subTypeBuf = st.typeBuf[:0], st.subTypeBuf[:0]
   233→		cluster, host := clusterDefault, ""
   234→		for {
   235→			key, val, err := dec.NextTag()
   236→			if err != nil {
   237→				return err
   238→			}
   239→			if key == nil {
   240→				break
   241→			}
   242→
   243→			// The go compiler optimizes string([]byte{...}) == "...":
   244→			switch string(key) {
   245→			case "cluster":
   246→				if string(val) == prevCluster {
   247→					cluster = prevCluster
   248→				} else {
   249→					cluster = string(val)
   250→					lvl = nil
   251→				}
   252→			case "hostname", "host":
   253→				if string(val) == prevHost {
   254→					host = prevHost
   255→				} else {
   256→					host = string(val)
   257→					lvl = nil
   258→				}
   259→			case "type":
   260→				if string(val) == "node" {
   261→					break
   262→				}
   263→
   264→				// We cannot be sure that the "type" tag comes before the "type-id" tag:
   265→				if len(st.typeBuf) == 0 {
   266→					st.typeBuf = append(st.typeBuf, val...)
   267→				} else {
   268→					st.typeBuf = reorder(st.typeBuf, val)
   269→				}
   270→			case "type-id":
   271→				st.typeBuf = append(st.typeBuf, val...)
   272→			case "subtype":
   273→				// We cannot be sure that the "subtype" tag comes before the "stype-id" tag:
   274→				if len(st.subTypeBuf) == 0 {
   275→					st.subTypeBuf = append(st.subTypeBuf, val...)
   276→				} else {
   277→					st.subTypeBuf = reorder(st.subTypeBuf, val)
   278→				}
   279→			case "stype-id":
   280→				st.subTypeBuf = append(st.subTypeBuf, val...)
   281→			default:
   282→			}
   283→		}
   284→
   285→		// If the cluster or host changed, the lvl was set to nil
   286→		if lvl == nil {
   287→			st.selector = st.selector[:2]
   288→			st.selector[0], st.selector[1] = cluster, host
   289→			lvl = ms.GetLevel(st.selector)
   290→			prevCluster, prevHost = cluster, host
   291→		}
   292→
   293→		// subtypes: cache []byte→string conversions; messages in a batch typically
   294→		// share the same type/subtype so the hit rate is very high.
   295→		st.selector = st.selector[:0]
   296→		if len(st.typeBuf) > 0 {
   297→			if !bytes.Equal(st.typeBuf, st.prevTypeBytes) {
   298→				st.prevTypeBytes = append(st.prevTypeBytes[:0], st.typeBuf...)
   299→				st.prevTypeStr = string(st.typeBuf)
   300→			}
   301→			st.selector = append(st.selector, st.prevTypeStr)
   302→			if len(st.subTypeBuf) > 0 {
   303→				if !bytes.Equal(st.subTypeBuf, st.prevSubTypeBytes) {
   304→					st.prevSubTypeBytes = append(st.prevSubTypeBytes[:0], st.subTypeBuf...)
   305→					st.prevSubTypeStr = string(st.subTypeBuf)
   306→				}
   307→				st.selector = append(st.selector, st.prevSubTypeStr)
   308→			}
   309→		}
   310→
   311→		for {
   312→			key, val, err := dec.NextField()
   313→			if err != nil {
   314→				return err
   315→			}
   316→
   317→			if key == nil {
   318→				break
   319→			}
   320→
   321→			if string(key) != "value" {
   322→				return fmt.Errorf("host %s: unknown field: '%s' (value: %#v)", host, string(key), val)
   323→			}
   324→
   325→			if val.Kind() == lineprotocol.Float {
   326→				metric.Value = schema.Float(val.FloatV())
   327→			} else if val.Kind() == lineprotocol.Int {
   328→				metric.Value = schema.Float(val.IntV())
   329→			} else if val.Kind() == lineprotocol.Uint {
   330→				metric.Value = schema.Float(val.UintV())
   331→			} else {
   332→				return fmt.Errorf("host %s: unsupported value type in message: %s", host, val.Kind().String())
   333→			}
   334→		}
   335→
   336→		if t, err = dec.Time(lineprotocol.Second, t); err != nil {
   337→			t = time.Now()
   338→			if t, err = dec.Time(lineprotocol.Millisecond, t); err != nil {
   339→				t = time.Now()
   340→				if t, err = dec.Time(lineprotocol.Microsecond, t); err != nil {
   341→					t = time.Now()
   342→					if t, err = dec.Time(lineprotocol.Nanosecond, t); err != nil {
   343→						return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
   344→					}
   345→				}
   346→			}
   347→		}
   348→
   349→		if err != nil {
   350→			return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
   351→		}
   352→
   353→		time := t.Unix()
   354→
   355→		if Keys.Checkpoints.FileFormat == "wal" {
   356→			msg := &WALMessage{
   357→				MetricName: string(st.metricBuf),
   358→				Cluster:    cluster,
   359→				Node:       host,
   360→				Selector:   append([]string{}, st.selector...),
   361→				Value:      metric.Value,
   362→				Timestamp:  time,
   363→			}
   364→			if !SendWALMessage(msg) {
   365→				// WAL shard channel full — metric is written to memory store but not WAL.
   366→				// Next binary snapshot will capture it.
   367→				if dropped := walDropped.Add(1); dropped%10000 == 1 {
   368→					cclog.Warnf("[METRICSTORE]> WAL shard channel full, dropped %d messages (data safe in memory, next snapshot will capture)", dropped)
   369→				}
   370→			}
   371→		}
   372→
   373→		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
   374→			cclog.Warnf("write error for host %s metric %s at ts %d: %s", host, string(st.metricBuf), time, err.Error())
   375→			continue
   376→		}
   377→	}
   378→	return nil
   379→}
   380→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides an efficient in-memory time-series metric storage system
     7→// with support for hierarchical data organization, checkpointing, and archiving.
     8→//
     9→// The package organizes metrics in a tree structure (cluster → host → component) and
    10→// provides concurrent read/write access to metric data with configurable aggregation strategies.
    11→// Background goroutines handle periodic checkpointing (JSON or Avro format), archiving old data,
    12→// and enforcing retention policies.
    13→//
    14→// Key features:
    15→//   - In-memory metric storage with configurable retention
    16→//   - Hierarchical data organization (selectors)
    17→//   - Concurrent checkpoint/archive workers
    18→//   - Support for sum and average aggregation
    19→//   - NATS integration for metric ingestion
    20→package metricstore
    21→
    22→import (
    23→	"bytes"
    24→	"context"
    25→	"encoding/json"
    26→	"errors"
    27→	"fmt"
    28→	"runtime"
    29→	"runtime/debug"
    30→	"slices"
    31→	"sync"
    32→	"time"
    33→
    34→	"github.com/ClusterCockpit/cc-backend/internal/config"
    35→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    36→	"github.com/ClusterCockpit/cc-lib/v2/resampler"
    37→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    38→	"github.com/ClusterCockpit/cc-lib/v2/util"
    39→)
    40→
    41→// GlobalState holds the global state for the metric store with thread-safe access.
    42→type GlobalState struct {
    43→	mu                sync.RWMutex
    44→	lastRetentionTime int64
    45→	selectorsExcluded bool
    46→}
    47→
    48→var (
    49→	singleton  sync.Once
    50→	msInstance *MemoryStore
    51→	// shutdownFunc stores the context cancellation function created in Init
    52→	// and is called during Shutdown to cancel all background goroutines
    53→	shutdownFunc   context.CancelFunc
    54→	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
    55→	// Create a global instance
    56→	state = &GlobalState{}
    57→)
    58→
    59→// NodeProvider provides information about nodes currently in use by running jobs.
    60→//
    61→// This interface allows metricstore to query job information without directly
    62→// depending on the repository package, breaking the import cycle.
    63→//
    64→// Implementations should return nodes that are actively processing jobs started
    65→// before the given timestamp. These nodes will be excluded from retention-based
    66→// garbage collection to prevent data loss for jobs that are still running or
    67→// recently completed.
    68→type NodeProvider interface {
    69→	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
    70→	// that are currently in use by jobs that started before the given timestamp.
    71→	//
    72→	// Parameters:
    73→	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
    74→	//
    75→	// Returns:
    76→	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
    77→	//   - Error if the query fails
    78→	GetUsedNodes(ts int64) (map[string][]string, error)
    79→}
    80→
    81→// Metric represents a single metric data point to be written to the store.
    82→type Metric struct {
    83→	Name  string
    84→	Value schema.Float
    85→	// MetricConfig contains frequency and aggregation settings for this metric.
    86→	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
    87→	MetricConfig MetricConfig
    88→}
    89→
    90→// MemoryStore is the main in-memory time-series metric storage implementation.
    91→//
    92→// It organizes metrics in a hierarchical tree structure where each level represents
    93→// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
    94→// store multiple metrics as time-series buffers.
    95→//
    96→// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
    97→// All public methods are safe for concurrent use.
    98→type MemoryStore struct {
    99→	Metrics      map[string]MetricConfig
   100→	root         Level
   101→	nodeProvider NodeProvider
   102→}
   103→
   104→// Init initializes the metric store from configuration and starts background workers.
   105→//
   106→// This function must be called exactly once before any other metricstore operations.
   107→// It performs the following initialization steps:
   108→//  1. Validates and decodes the metric store configuration
   109→//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
   110→//  3. Loads metric configurations from all registered clusters
   111→//  4. Restores checkpoints within the retention window
   112→//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
   113→//  6. Optionally subscribes to NATS for real-time metric ingestion
   114→//
   115→// Parameters:
   116→//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig)
   117→//   - wg: WaitGroup that will be incremented for each background goroutine started
   118→//
   119→// The function will call cclog.Fatal on critical errors during initialization.
   120→// Use Shutdown() to cleanly stop all background workers started by Init().
   121→//
   122→// Note: Signal handling must be implemented by the caller. Call Shutdown() when
   123→// receiving termination signals to ensure checkpoint data is persisted.
   124→func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
   125→	startupTime := time.Now()
   126→
   127→	if rawConfig != nil {
   128→		config.Validate(configSchema, rawConfig)
   129→		dec := json.NewDecoder(bytes.NewReader(rawConfig))
   130→		dec.DisallowUnknownFields()
   131→		if err := dec.Decode(&Keys); err != nil {
   132→			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
   133→		}
   134→	}
   135→
   136→	// Set NumWorkers from config or use default
   137→	if Keys.NumWorkers <= 0 {
   138→		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
   139→	}
   140→	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
   141→
   142→	// Pass the config.MetricStoreKeys
   143→	InitMetrics(metrics)
   144→
   145→	ms := GetMemoryStore()
   146→
   147→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   148→	if err != nil {
   149→		cclog.Fatal(err)
   150→	}
   151→
   152→	restoreFrom := startupTime.Add(-d)
   153→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   154→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   155→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   156→	if err != nil {
   157→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   158→	} else {
   159→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   160→	}
   161→
   162→	// Try to use less memory by forcing a GC run here and then
   163→	// lowering the target percentage. The default of 100 means
   164→	// that only once the ratio of new allocations execeds the
   165→	// previously active heap, a GC is triggered.
   166→	// Forcing a GC here will set the "previously active heap"
   167→	// to a minumum.
   168→	// runtime.GC()
   169→
   170→	ctx, shutdown := context.WithCancel(context.Background())
   171→
   172→	Retention(wg, ctx)
   173→	Checkpointing(wg, ctx)
   174→	CleanUp(wg, ctx)
   175→	WALStaging(wg, ctx)
   176→	MemoryUsageTracker(wg, ctx)
   177→
   178→	// Note: Signal handling has been removed from this function.
   179→	// The caller is responsible for handling shutdown signals and calling
   180→	// the shutdown() function when appropriate.
   181→	// Store the shutdown function for later use by Shutdown()
   182→	shutdownFuncMu.Lock()
   183→	shutdownFunc = shutdown
   184→	shutdownFuncMu.Unlock()
   185→
   186→	if Keys.Subscriptions != nil {
   187→		wg.Go(func() {
   188→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   189→				cclog.Fatal(err)
   190→			}
   191→		})
   192→	}
   193→}
   194→
   195→// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
   196→//
   197→// This function must be called before GetMemoryStore() and can only be called once due to
   198→// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
   199→//
   200→// Parameters:
   201→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
   202→//
   203→// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
   204→//
   205→// After this call, the global msInstance is ready for use via GetMemoryStore().
   206→func InitMetrics(metrics map[string]MetricConfig) {
   207→	singleton.Do(func() {
   208→		offset := 0
   209→		for key, cfg := range metrics {
   210→			if cfg.Frequency == 0 {
   211→				panic("[METRICSTORE]> invalid frequency")
   212→			}
   213→
   214→			metrics[key] = MetricConfig{
   215→				Frequency:   cfg.Frequency,
   216→				Aggregation: cfg.Aggregation,
   217→				offset:      offset,
   218→			}
   219→			offset += 1
   220→		}
   221→
   222→		msInstance = &MemoryStore{
   223→			root: Level{
   224→				metrics:  make([]*buffer, len(metrics)),
   225→				children: make(map[string]*Level),
   226→			},
   227→			Metrics: metrics,
   228→		}
   229→	})
   230→}
   231→
   232→// GetMemoryStore returns the singleton MemoryStore instance.
   233→//
   234→// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
   235→//
   236→// This function is safe for concurrent use after initialization.
   237→func GetMemoryStore() *MemoryStore {
   238→	if msInstance == nil {
   239→		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
   240→	}
   241→
   242→	return msInstance
   243→}
   244→
   245→func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
   246→	if metric, ok := ms.Metrics[metricName]; ok {
   247→		return metric.Frequency, nil
   248→	}
   249→	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
   250→}
   251→
   252→// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
   253→// This must be called during initialization to provide job state information
   254→// for selective buffer retention during Free operations.
   255→// If not set, the Free function will fall back to freeing all buffers.
   256→func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
   257→	ms.nodeProvider = provider
   258→}
   259→
   260→// Shutdown performs a graceful shutdown of the metric store.
   261→//
   262→// This function cancels all background goroutines started by Init() and writes
   263→// a final checkpoint to disk before returning. It should be called when the
   264→// application receives a termination signal.
   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	totalStart := time.Now()
   275→
   276→	shutdownFuncMu.Lock()
   277→	defer shutdownFuncMu.Unlock()
   278→	if shutdownFunc != nil {
   279→		shutdownFunc()
   280→	}
   281→	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))
   282→
   283→	if Keys.Checkpoints.FileFormat == "wal" {
   284→		for _, ch := range walShardChs {
   285→			close(ch)
   286→		}
   287→		drainStart := time.Now()
   288→		WaitForWALStagingDrain()
   289→		cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
   290→	}
   291→
   292→	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
   293→	checkpointStart := time.Now()
   294→	var files int
   295→	var err error
   296→
   297→	ms := GetMemoryStore()
   298→
   299→	lastCheckpointMu.Lock()
   300→	from := lastCheckpoint
   301→	lastCheckpointMu.Unlock()
   302→
   303→	if Keys.Checkpoints.FileFormat == "wal" {
   304→		// WAL files are deleted per-host inside ToCheckpointWAL workers.
   305→		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   306→	} else {
   307→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   308→	}
   309→
   310→	if err != nil {
   311→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
   312→	}
   313→	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
   314→}
   315→
   316→// Retention starts a background goroutine that periodically frees old metric data.
   317→//
   318→// This worker runs at half the retention interval and calls Free() to remove buffers
   319→// older than the configured retention time. It respects the NodeProvider to preserve
   320→// data for nodes with active jobs.
   321→//
   322→// Parameters:
   323→//   - wg: WaitGroup to signal completion when context is cancelled
   324→//   - ctx: Context for cancellation signal
   325→//
   326→// The goroutine exits when ctx is cancelled.
   327→func Retention(wg *sync.WaitGroup, ctx context.Context) {
   328→	ms := GetMemoryStore()
   329→
   330→	wg.Go(func() {
   331→		d, err := time.ParseDuration(Keys.RetentionInMemory)
   332→		if err != nil {
   333→			cclog.Fatal(err)
   334→		}
   335→		if d <= 0 {
   336→			return
   337→		}
   338→
   339→		tickInterval := d / 2
   340→		if tickInterval <= 0 {
   341→			return
   342→		}
   343→		ticker := time.NewTicker(tickInterval)
   344→		defer ticker.Stop()
   345→
   346→		for {
   347→			select {
   348→			case <-ctx.Done():
   349→				return
   350→			case <-ticker.C:
   351→				state.mu.Lock()
   352→
   353→				t := time.Now().Add(-d)
   354→
   355→				state.lastRetentionTime = t.Unix()
   356→
   357→				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
   358→
   359→				freed, err := Free(ms, t)
   360→				if err != nil {
   361→					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
   362→				} else {
   363→					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
   364→				}
   365→
   366→				state.mu.Unlock()
   367→
   368→				// Clean up the buffer pool
   369→				bufferPool.Clean(state.lastRetentionTime)
   370→			}
   371→		}
   372→	})
   373→}
   374→
   375→// MemoryUsageTracker starts a background goroutine that monitors memory usage.
   376→//
   377→// This worker checks actual process memory usage (via runtime.MemStats) periodically
   378→// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
   379→// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
   380→// performance issues.
   381→//
   382→// The tracker logs both actual memory usage (heap allocated) and metric data size for
   383→// visibility into memory overhead from Go runtime structures and allocations.
   384→//
   385→// Parameters:
   386→//   - wg: WaitGroup to signal completion when context is cancelled
   387→//   - ctx: Context for cancellation signal
   388→//
   389→// The goroutine exits when ctx is cancelled.
   390→func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
   391→	ms := GetMemoryStore()
   392→
   393→	wg.Go(func() {
   394→		normalInterval := DefaultMemoryUsageTrackerInterval
   395→		fastInterval := 30 * time.Second
   396→
   397→		if normalInterval <= 0 {
   398→			return
   399→		}
   400→
   401→		ticker := time.NewTicker(normalInterval)
   402→		defer ticker.Stop()
   403→		currentInterval := normalInterval
   404→
   405→		for {
   406→			select {
   407→			case <-ctx.Done():
   408→				return
   409→			case <-ticker.C:
   410→				var mem runtime.MemStats
   411→				runtime.ReadMemStats(&mem)
   412→				actualMemoryGB := float64(mem.Alloc) / 1e9
   413→				metricDataGB := ms.SizeInGB()
   414→				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
   415→
   416→				freedExcluded := 0
   417→				freedEmergency := 0
   418→				var err error
   419→
   420→				state.mu.RLock()
   421→				lastRetention := state.lastRetentionTime
   422→				selectorsExcluded := state.selectorsExcluded
   423→				state.mu.RUnlock()
   424→
   425→				if lastRetention != 0 && selectorsExcluded {
   426→					freedExcluded, err = ms.Free(nil, lastRetention)
   427→					if err != nil {
   428→						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
   429→					}
   430→
   431→					if freedExcluded > 0 {
   432→						debug.FreeOSMemory()
   433→						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
   434→					}
   435→				}
   436→
   437→				runtime.ReadMemStats(&mem)
   438→				actualMemoryGB = float64(mem.Alloc) / 1e9
   439→
   440→				if actualMemoryGB > float64(Keys.MemoryCap) {
   441→					cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)
   442→
   443→					// Use progressive time-based Free with increasing threshold
   444→					// instead of ForceFree loop — fewer tree traversals, more effective
   445→					d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
   446→					if parseErr != nil {
   447→						cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
   448→					} else {
   449→						thresholds := []float64{0.75, 0.5, 0.25}
   450→						for _, fraction := range thresholds {
   451→							threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
   452→							freed, freeErr := ms.Free(nil, threshold.Unix())
   453→							if freeErr != nil {
   454→								cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
   455→							}
   456→							freedEmergency += freed
   457→
   458→							bufferPool.Clear()
   459→							runtime.GC()
   460→							runtime.ReadMemStats(&mem)
   461→							actualMemoryGB = float64(mem.Alloc) / 1e9
   462→
   463→							if actualMemoryGB < float64(Keys.MemoryCap) {
   464→								break
   465→							}
   466→						}
   467→					}
   468→
   469→					bufferPool.Clear()
   470→					debug.FreeOSMemory()
   471→
   472→					runtime.ReadMemStats(&mem)
   473→					actualMemoryGB = float64(mem.Alloc) / 1e9
   474→
   475→					if actualMemoryGB >= float64(Keys.MemoryCap) {
   476→						cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
   477→					} else {
   478→						cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
   479→					}
   480→				}
   481→
   482→				// Adaptive ticker: check more frequently when memory is high
   483→				memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
   484→				if memoryRatio > 0.8 && currentInterval != fastInterval {
   485→					ticker.Reset(fastInterval)
   486→					currentInterval = fastInterval
   487→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
   488→				} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
   489→					ticker.Reset(normalInterval)
   490→					currentInterval = normalInterval
   491→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
   492→				}
   493→			}
   494→		}
   495→	})
   496→}
   497→
   498→// Free removes metric data older than the given time while preserving data for active nodes.
   499→//
   500→// This function implements intelligent retention by consulting the NodeProvider (if configured)
   501→// to determine which nodes are currently in use by running jobs. Data for these nodes is
   502→// preserved even if older than the retention time.
   503→//
   504→// Parameters:
   505→//   - ms: The MemoryStore instance
   506→//   - t: Time threshold - buffers with data older than this will be freed
   507→//
   508→// Returns:
   509→//   - Number of buffers freed
   510→//   - Error if NodeProvider query fails
   511→//
   512→// Behavior:
   513→//   - If no NodeProvider is set: frees all buffers older than t
   514→//   - If NodeProvider returns empty map: frees all buffers older than t
   515→//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
   516→func Free(ms *MemoryStore, t time.Time) (int, error) {
   517→	// If no NodeProvider is configured, free all buffers older than t
   518→	if ms.nodeProvider == nil {
   519→		return ms.Free(nil, t.Unix())
   520→	}
   521→
   522→	excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
   523→	if err != nil {
   524→		return 0, err
   525→	}
   526→
   527→	switch lenMap := len(excludeSelectors); lenMap {
   528→
   529→	// If the length of the map returned by GetUsedNodes() is 0,
   530→	// then use default Free method with nil selector
   531→	case 0:
   532→		state.selectorsExcluded = false
   533→		return ms.Free(nil, t.Unix())
   534→
   535→	// Else formulate selectors, exclude those from the map
   536→	// and free the rest of the selectors
   537→	default:
   538→		state.selectorsExcluded = true
   539→		selectors := GetSelectors(ms, excludeSelectors)
   540→		return FreeSelected(ms, selectors, t)
   541→	}
   542→}
   543→
   544→// FreeSelected frees buffers for specific selectors while preserving others.
   545→//
   546→// This function is used when we want to retain some specific nodes beyond the retention time.
   547→// It iterates through the provided selectors and frees their associated buffers.
   548→//
   549→// Parameters:
   550→//   - ms: The MemoryStore instance
   551→//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
   552→//   - t: Time threshold for freeing buffers
   553→//
   554→// Returns the total number of buffers freed and any error encountered.
   555→func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
   556→	freed := 0
   557→
   558→	for _, selector := range selectors {
   559→
   560→		freedBuffers, err := ms.Free(selector, t.Unix())
   561→		if err != nil {
   562→			cclog.Errorf("error while freeing selected buffers: %#v", err)
   563→		}
   564→		freed += freedBuffers
   565→
   566→	}
   567→
   568→	return freed, nil
   569→}
   570→
   571→// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
   572→//
   573→// This function generates a list of selectors whose buffers should be freed by excluding
   574→// selectors that correspond to nodes currently in use by running jobs.
   575→//
   576→// Parameters:
   577→//   - ms: The MemoryStore instance
   578→//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
   579→//
   580→// Returns a list of selectors ([]string paths) that can be safely freed.
   581→//
   582→// Example:
   583→//
   584→//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
   585→//	and excludeSelectors contains {"emmy": ["node001"]},
   586→//	then only [["emmy", "node002"]] is returned.
   587→func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
   588→	allSelectors := ms.GetPaths(2)
   589→
   590→	filteredSelectors := make([][]string, 0, len(allSelectors))
   591→
   592→	for _, path := range allSelectors {
   593→		if len(path) < 2 {
   594→			continue
   595→		}
   596→
   597→		key := path[0]   // The "Key" (Level 1)
   598→		value := path[1] // The "Value" (Level 2)
   599→
   600→		exclude := false
   601→
   602→		// Check if the key exists in our exclusion map
   603→		if excludedValues, exists := excludeSelectors[key]; exists {
   604→			// The key exists, now check if the specific value is in the exclusion list
   605→			if slices.Contains(excludedValues, value) {
   606→				exclude = true
   607→			}
   608→		}
   609→
   610→		if !exclude {
   611→			filteredSelectors = append(filteredSelectors, path)
   612→		}
   613→	}
   614→
   615→	return filteredSelectors
   616→}
   617→
   618→// GetPaths returns a list of lists (paths) to the specified depth.
   619→func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
   620→	var results [][]string
   621→
   622→	// Start recursion. Initial path is empty.
   623→	// We treat Root as depth 0.
   624→	ms.root.collectPaths(0, targetDepth, []string{}, &results)
   625→
   626→	return results
   627→}
   628→
   629→// Write all values in `metrics` to the level specified by `selector` for time `ts`.
   630→// Look at `findLevelOrCreate` for how selectors work.
   631→func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
   632→	var ok bool
   633→	for i, metric := range metrics {
   634→		if metric.MetricConfig.Frequency == 0 {
   635→			metric.MetricConfig, ok = m.Metrics[metric.Name]
   636→			if !ok {
   637→				cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
   638→				metric.MetricConfig.Frequency = 0
   639→			}
   640→			metrics[i] = metric
   641→		}
   642→	}
   643→
   644→	return m.WriteToLevel(&m.root, selector, ts, metrics)
   645→}
   646→
   647→func (m *MemoryStore) GetLevel(selector []string) *Level {
   648→	return m.root.findLevelOrCreate(selector, len(m.Metrics))
   649→}
   650→
   651→// WriteToLevel assumes that `minfo` in `metrics` is filled in
   652→func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
   653→	l = l.findLevelOrCreate(selector, len(m.Metrics))
   654→	l.lock.Lock()
   655→	defer l.lock.Unlock()
   656→
   657→	for _, metric := range metrics {
   658→		if metric.MetricConfig.Frequency == 0 {
   659→			continue
   660→		}
   661→
   662→		b := l.metrics[metric.MetricConfig.offset]
   663→		if b == nil {
   664→			// First write to this metric and level
   665→			b = newBuffer(ts, metric.MetricConfig.Frequency)
   666→			l.metrics[metric.MetricConfig.offset] = b
   667→		}
   668→
   669→		nb, err := b.write(ts, metric.Value)
   670→		if err != nil {
   671→			return err
   672→		}
   673→
   674→		// Last write created a new buffer...
   675→		if b != nb {
   676→			l.metrics[metric.MetricConfig.offset] = nb
   677→		}
   678→	}
   679→	return nil
   680→}
   681→
   682→// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
   683→// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
   684→// The second and third return value are the actual from/to for the data. Those can be different from
   685→// the range asked for if no data was available.
   686→func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
   687→	if from > to {
   688→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
   689→	}
   690→
   691→	minfo, ok := m.Metrics[metric]
   692→	if !ok {
   693→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
   694→	}
   695→
   696→	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
   697→
   698→	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
   699→		cdata, cfrom, cto, err := b.read(from, to, data)
   700→		if err != nil {
   701→			return err
   702→		}
   703→
   704→		if n == 0 {
   705→			from, to = cfrom, cto
   706→		} else if from != cfrom || to != cto || len(data) != len(cdata) {
   707→			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
   708→			if missingfront != 0 {
   709→				return ErrDataDoesNotAlign
   710→			}
   711→
   712→			newlen := len(cdata) - missingback
   713→			if newlen < 1 {
   714→				return ErrDataDoesNotAlign
   715→			}
   716→			cdata = cdata[0:newlen]
   717→			if len(cdata) != len(data) {
   718→				return ErrDataDoesNotAlign
   719→			}
   720→
   721→			from, to = cfrom, cto
   722→		}
   723→
   724→		data = cdata
   725→		n += 1
   726→		return nil
   727→	})
   728→
   729→	if err != nil {
   730→		return nil, 0, 0, 0, err
   731→	} else if n == 0 {
   732→		return nil, 0, 0, 0, ErrNoHostOrMetric
   733→	} else if n > 1 {
   734→		if minfo.Aggregation == AvgAggregation {
   735→			normalize := 1. / schema.Float(n)
   736→			for i := 0; i < len(data); i++ {
   737→				data[i] *= normalize
   738→			}
   739→		} else if minfo.Aggregation != SumAggregation {
   740→			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
   741→		}
   742→	}
   743→
   744→	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
   745→	if err != nil {
   746→		return nil, 0, 0, 0, err
   747→	}
   748→
   749→	return data, from, to, resolution, nil
   750→}
   751→
   752→// Free releases all buffers for the selected level and all its children that
   753→// contain only values older than `t`.
   754→func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
   755→	return m.GetLevel(selector).free(t)
   756→}
   757→
   758→// ForceFree unconditionally removes the oldest buffer from each metric chain.
   759→func (m *MemoryStore) ForceFree() (int, error) {
   760→	return m.GetLevel(nil).forceFree()
   761→}
   762→
   763→func (m *MemoryStore) FreeAll() error {
   764→	m.root.lock.Lock()
   765→	defer m.root.lock.Unlock()
   766→
   767→	for k := range m.root.children {
   768→		delete(m.root.children, k)
   769→	}
   770→
   771→	return nil
   772→}
   773→
   774→func (m *MemoryStore) SizeInBytes() int64 {
   775→	return m.root.sizeInBytes()
   776→}
   777→
   778→func (m *MemoryStore) SizeInGB() float64 {
   779→	return float64(m.root.sizeInBytes()) / 1e9
   780→}
   781→
   782→// ListChildren , given a selector, returns a list of all children of the level
   783→// selected.
   784→func (m *MemoryStore) ListChildren(selector []string) []string {
   785→	lvl := &m.root
   786→	for lvl != nil && len(selector) != 0 {
   787→		lvl.lock.RLock()
   788→		next := lvl.children[selector[0]]
   789→		lvl.lock.RUnlock()
   790→		lvl = next
   791→		selector = selector[1:]
   792→	}
   793→
   794→	if lvl == nil {
   795→		return nil
   796→	}
   797→
   798→	lvl.lock.RLock()
   799→	defer lvl.lock.RUnlock()
   800→
   801→	children := make([]string, 0, len(lvl.children))
   802→	for child := range lvl.children {
   803→		children = append(children, child)
   804→	}
   805→
   806→	return children
   807→}
   808→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements checkpoint persistence for the in-memory metric store.
     7→//
     8→// Checkpoints enable graceful restarts by periodically saving in-memory metric
     9→// data to disk in JSON or binary format. The checkpoint system:
    10→//
    11→// Key Features:
    12→//   - Periodic background checkpointing via the Checkpointing() worker
    13→//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
    14→//   - Parallel checkpoint creation and loading using worker pools
    15→//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
    16→//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
    17→//   - Only saves unarchived data (archived data is already persisted elsewhere)
    18→//   - GC optimization during loading to prevent excessive heap growth
    19→//
    20→// Checkpoint Workflow:
    21→//  1. Init() loads checkpoints within retention window at startup
    22→//  2. Checkpointing() worker periodically saves new data
    23→//  3. Shutdown() writes final checkpoint before exit
    24→//
    25→// File Organization:
    26→//
    27→//	checkpoints/
    28→//	  cluster1/
    29→//	    host001/
    30→//	      1234567890.json  (JSON format: full subtree snapshot)
    31→//	      1234567890.bin   (binary format: full subtree snapshot)
    32→//	      current.wal      (WAL format: append-only per-entry log)
    33→//	    host002/
    34→//	      ...
    35→package metricstore
    36→
    37→import (
    38→	"bufio"
    39→	"context"
    40→	"encoding/json"
    41→	"errors"
    42→	"fmt"
    43→	"io/fs"
    44→	"os"
    45→	"path"
    46→	"path/filepath"
    47→	"sort"
    48→	"strconv"
    49→	"strings"
    50→	"sync"
    51→	"sync/atomic"
    52→	"time"
    53→
    54→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    55→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    56→)
    57→
    58→const (
    59→	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
    60→	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
    61→	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
    62→)
    63→
    64→// CheckpointMetrics represents metric data in a checkpoint file.
    65→// Whenever the structure changes, update MarshalJSON as well!
    66→type CheckpointMetrics struct {
    67→	Data      []schema.Float `json:"data"`
    68→	Frequency int64          `json:"frequency"`
    69→	Start     int64          `json:"start"`
    70→}
    71→
    72→// CheckpointFile represents the hierarchical structure of a checkpoint file.
    73→// It mirrors the Level tree structure from the MemoryStore.
    74→type CheckpointFile struct {
    75→	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
    76→	Children map[string]*CheckpointFile    `json:"children"`
    77→	From     int64                         `json:"from"`
    78→	To       int64                         `json:"to"`
    79→}
    80→
    81→// lastCheckpoint tracks the timestamp of the last checkpoint creation.
    82→var (
    83→	lastCheckpoint   time.Time
    84→	lastCheckpointMu sync.Mutex
    85→)
    86→
    87→// Checkpointing starts a background worker that periodically saves metric data to disk.
    88→//
    89→// Checkpoints are written every 12 hours (hardcoded).
    90→//
    91→// Format behaviour:
    92→//   - "json": Periodic checkpointing every checkpointInterval
    93→//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
    94→func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
    95→	lastCheckpointMu.Lock()
    96→	lastCheckpoint = time.Now()
    97→	lastCheckpointMu.Unlock()
    98→
    99→	ms := GetMemoryStore()
   100→
   101→	wg.Go(func() {
   102→
   103→		d := 12 * time.Hour // default checkpoint interval
   104→		if Keys.CheckpointInterval != "" {
   105→			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
   106→			if err != nil {
   107→				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
   108→			} else {
   109→				d = parsed
   110→			}
   111→		}
   112→
   113→		ticker := time.NewTicker(d)
   114→		defer ticker.Stop()
   115→
   116→		for {
   117→			select {
   118→			case <-ctx.Done():
   119→				return
   120→			case <-ticker.C:
   121→				lastCheckpointMu.Lock()
   122→				from := lastCheckpoint
   123→				lastCheckpointMu.Unlock()
   124→
   125→				now := time.Now()
   126→				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
   127→
   128→				if Keys.Checkpoints.FileFormat == "wal" {
   129→					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   130→					if err != nil {
   131→						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
   132→					} else {
   133→						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
   134→						lastCheckpointMu.Lock()
   135→						lastCheckpoint = now
   136→						lastCheckpointMu.Unlock()
   137→						// Rotate WAL files for successfully checkpointed hosts.
   138→						RotateWALFiles(hostDirs)
   139→					}
   140→				} else {
   141→					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   142→					if err != nil {
   143→						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
   144→					} else {
   145→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
   146→						lastCheckpointMu.Lock()
   147→						lastCheckpoint = now
   148→						lastCheckpointMu.Unlock()
   149→					}
   150→				}
   151→			}
   152→		}
   153→	})
   154→}
   155→
   156→// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
   157→//
   158→// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
   159→// This method manually constructs JSON to avoid allocations and interface conversions.
   160→func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
   161→	buf := make([]byte, 0, 128+len(cm.Data)*8)
   162→	buf = append(buf, `{"frequency":`...)
   163→	buf = strconv.AppendInt(buf, cm.Frequency, 10)
   164→	buf = append(buf, `,"start":`...)
   165→	buf = strconv.AppendInt(buf, cm.Start, 10)
   166→	buf = append(buf, `,"data":[`...)
   167→	for i, x := range cm.Data {
   168→		if i != 0 {
   169→			buf = append(buf, ',')
   170→		}
   171→		if x.IsNaN() {
   172→			buf = append(buf, `null`...)
   173→		} else {
   174→			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
   175→		}
   176→	}
   177→	buf = append(buf, `]}`...)
   178→	return buf, nil
   179→}
   180→
   181→// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format).
   182→//
   183→// Metrics at root and cluster levels are skipped. One file per host is created.
   184→// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
   185→// at a time, allowing concurrent writes/reads to other hosts.
   186→//
   187→// Returns the number of checkpoint files created and any errors encountered.
   188→func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
   189→	// Pre-calculate capacity by counting cluster/host pairs
   190→	m.root.lock.RLock()
   191→	totalHosts := 0
   192→	for _, l1 := range m.root.children {
   193→		l1.lock.RLock()
   194→		totalHosts += len(l1.children)
   195→		l1.lock.RUnlock()
   196→	}
   197→	m.root.lock.RUnlock()
   198→
   199→	levels := make([]*Level, 0, totalHosts)
   200→	selectors := make([][]string, 0, totalHosts)
   201→
   202→	m.root.lock.RLock()
   203→	for sel1, l1 := range m.root.children {
   204→		l1.lock.RLock()
   205→		for sel2, l2 := range l1.children {
   206→			levels = append(levels, l2)
   207→			selectors = append(selectors, []string{sel1, sel2})
   208→		}
   209→		l1.lock.RUnlock()
   210→	}
   211→	m.root.lock.RUnlock()
   212→
   213→	type workItem struct {
   214→		level    *Level
   215→		dir      string
   216→		selector []string
   217→	}
   218→
   219→	n, errs := int32(0), int32(0)
   220→
   221→	var wg sync.WaitGroup
   222→	wg.Add(Keys.NumWorkers)
   223→	work := make(chan workItem, Keys.NumWorkers*2)
   224→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   225→		go func() {
   226→			defer wg.Done()
   227→
   228→			for workItem := range work {
   229→				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
   230→					if err == ErrNoNewArchiveData {
   231→						continue
   232→					}
   233→
   234→					cclog.Errorf("[METRICSTORE]> error while checkpointing %#v: %s", workItem.selector, err.Error())
   235→					atomic.AddInt32(&errs, 1)
   236→				} else {
   237→					atomic.AddInt32(&n, 1)
   238→				}
   239→			}
   240→		}()
   241→	}
   242→
   243→	for i := 0; i < len(levels); i++ {
   244→		dir := path.Join(dir, path.Join(selectors[i]...))
   245→		work <- workItem{
   246→			level:    levels[i],
   247→			dir:      dir,
   248→			selector: selectors[i],
   249→		}
   250→	}
   251→
   252→	close(work)
   253→	wg.Wait()
   254→
   255→	if errs > 0 {
   256→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   257→	}
   258→	return int(n), nil
   259→}
   260→
   261→// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
   262→// Skips metrics that are already archived. Returns nil if no unarchived data exists.
   263→func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
   264→	l.lock.RLock()
   265→	defer l.lock.RUnlock()
   266→
   267→	retval := &CheckpointFile{
   268→		From:     from,
   269→		To:       to,
   270→		Metrics:  make(map[string]*CheckpointMetrics),
   271→		Children: make(map[string]*CheckpointFile),
   272→	}
   273→
   274→	for metric, minfo := range m.Metrics {
   275→		b := l.metrics[minfo.offset]
   276→		if b == nil {
   277→			continue
   278→		}
   279→
   280→		allArchived := true
   281→		b.iterFromTo(from, to, func(b *buffer) error {
   282→			if !b.archived {
   283→				allArchived = false
   284→				return fmt.Errorf("stop") // Early termination signal
   285→			}
   286→			return nil
   287→		})
   288→
   289→		if allArchived {
   290→			continue
   291→		}
   292→
   293→		data := make([]schema.Float, (to-from)/b.frequency+1)
   294→		data, start, end, err := b.read(from, to, data)
   295→		if err != nil {
   296→			return nil, err
   297→		}
   298→
   299→		for i := int((end - start) / b.frequency); i < len(data); i++ {
   300→			data[i] = schema.NaN
   301→		}
   302→
   303→		retval.Metrics[metric] = &CheckpointMetrics{
   304→			Frequency: b.frequency,
   305→			Start:     start,
   306→			Data:      data,
   307→		}
   308→	}
   309→
   310→	for name, child := range l.children {
   311→		val, err := child.toCheckpointFile(from, to, m)
   312→		if err != nil {
   313→			return nil, err
   314→		}
   315→
   316→		if val != nil {
   317→			retval.Children[name] = val
   318→		}
   319→	}
   320→
   321→	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
   322→		return nil, nil
   323→	}
   324→
   325→	return retval, nil
   326→}
   327→
   328→// toCheckpoint writes a Level's data to a JSON checkpoint file.
   329→// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.
   330→func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
   331→	cf, err := l.toCheckpointFile(from, to, m)
   332→	if err != nil {
   333→		return err
   334→	}
   335→
   336→	if cf == nil {
   337→		return ErrNoNewArchiveData
   338→	}
   339→
   340→	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
   341→	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   342→	if err != nil && os.IsNotExist(err) {
   343→		err = os.MkdirAll(dir, CheckpointDirPerms)
   344→		if err == nil {
   345→			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   346→		}
   347→	}
   348→	if err != nil {
   349→		return err
   350→	}
   351→	defer f.Close()
   352→
   353→	bw := bufio.NewWriter(f)
   354→	if err = json.NewEncoder(bw).Encode(cf); err != nil {
   355→		return err
   356→	}
   357→
   358→	return bw.Flush()
   359→}
   360→
   361→// enqueueCheckpointHosts traverses checkpoint directory and enqueues cluster/host pairs.
   362→// Returns error if directory structure is invalid.
   363→func enqueueCheckpointHosts(dir string, work chan<- [2]string) error {
   364→	clustersDir, err := os.ReadDir(dir)
   365→	if err != nil {
   366→		return err
   367→	}
   368→
   369→	for _, clusterDir := range clustersDir {
   370→		if !clusterDir.IsDir() {
   371→			return errors.New("[METRICSTORE]> expected only directories at first level of checkpoints/ directory")
   372→		}
   373→
   374→		hostsDir, err := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
   375→		if err != nil {
   376→			return err
   377→		}
   378→
   379→		for _, hostDir := range hostsDir {
   380→			if !hostDir.IsDir() {
   381→				return errors.New("[METRICSTORE]> expected only directories at second level of checkpoints/ directory")
   382→			}
   383→
   384→			work <- [2]string{clusterDir.Name(), hostDir.Name()}
   385→		}
   386→	}
   387→
   388→	return nil
   389→}
   390→
   391→// FromCheckpoint loads checkpoint files from disk into memory in parallel.
   392→//
   393→// Uses worker pool to load cluster/host combinations. Returns number of files
   394→// loaded and any errors.
   395→func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
   396→	var wg sync.WaitGroup
   397→	work := make(chan [2]string, Keys.NumWorkers*4)
   398→	n, errs := int32(0), int32(0)
   399→
   400→	wg.Add(Keys.NumWorkers)
   401→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   402→		go func() {
   403→			defer wg.Done()
   404→			for host := range work {
   405→				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
   406→				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
   407→				if err != nil {
   408→					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
   409→					atomic.AddInt32(&errs, 1)
   410→				}
   411→				atomic.AddInt32(&n, int32(nn))
   412→			}
   413→		}()
   414→	}
   415→
   416→	err := enqueueCheckpointHosts(dir, work)
   417→	close(work)
   418→	wg.Wait()
   419→
   420→	if err != nil {
   421→		return int(n), err
   422→	}
   423→
   424→	if errs > 0 {
   425→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   426→	}
   427→	return int(n), nil
   428→}
   429→
   430→// FromCheckpointFiles is the main entry point for loading checkpoints at startup.
   431→//
   432→// Creates checkpoint directory if it doesn't exist. This function must be called
   433→// before any writes or reads, and can only be called once.
   434→func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
   435→	if _, err := os.Stat(dir); os.IsNotExist(err) {
   436→		err := os.MkdirAll(dir, CheckpointDirPerms)
   437→		if err != nil {
   438→			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
   439→		}
   440→		cclog.Debugf("[METRICSTORE]> %#v Directory created successfully", dir)
   441→	}
   442→
   443→	return m.FromCheckpoint(dir, from)
   444→}
   445→
   446→func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
   447→	br := bufio.NewReader(f)
   448→	cf := &CheckpointFile{}
   449→	if err := json.NewDecoder(br).Decode(cf); err != nil {
   450→		return err
   451→	}
   452→
   453→	if cf.To != 0 && cf.To < from {
   454→		return nil
   455→	}
   456→
   457→	if err := l.loadFile(cf, m); err != nil {
   458→		return err
   459→	}
   460→
   461→	return nil
   462→}
   463→
   464→func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
   465→	for name, metric := range cf.Metrics {
   466→		n := len(metric.Data)
   467→		b := &buffer{
   468→			frequency: metric.Frequency,
   469→			start:     metric.Start,
   470→			data:      metric.Data[0:n:n],
   471→			prev:      nil,
   472→			next:      nil,
   473→			archived:  true,
   474→		}
   475→
   476→		minfo, ok := m.Metrics[name]
   477→		if !ok {
   478→			continue
   479→		}
   480→
   481→		prev := l.metrics[minfo.offset]
   482→		if prev == nil {
   483→			l.metrics[minfo.offset] = b
   484→		} else {
   485→			if prev.start > b.start {
   486→				return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start)
   487→			}
   488→
   489→			b.prev = prev
   490→			prev.next = b
   491→		}
   492→		l.metrics[minfo.offset] = b
   493→	}
   494→
   495→	if len(cf.Children) > 0 && l.children == nil {
   496→		l.children = make(map[string]*Level)
   497→	}
   498→
   499→	for sel, childCf := range cf.Children {
   500→		child, ok := l.children[sel]
   501→		if !ok {
   502→			child = &Level{
   503→				metrics:  make([]*buffer, len(m.Metrics)),
   504→				children: nil,
   505→			}
   506→			l.children[sel] = child
   507→		}
   508→
   509→		if err := child.loadFile(childCf, m); err != nil {
   510→			return err
   511→		}
   512→	}
   513→
   514→	return nil
   515→}
   516→
   517→// fromCheckpoint loads all checkpoint files (JSON, binary snapshot, WAL) for a
   518→// single host directory. Snapshot files are loaded first (sorted by timestamp),
   519→// then current.wal is replayed on top.
   520→func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {
   521→	direntries, err := os.ReadDir(dir)
   522→	if err != nil {
   523→		if os.IsNotExist(err) {
   524→			return 0, nil
   525→		}
   526→		return 0, err
   527→	}
   528→
   529→	allFiles := make([]fs.DirEntry, 0)
   530→	var walEntry fs.DirEntry
   531→	filesLoaded := 0
   532→
   533→	for _, e := range direntries {
   534→		if e.IsDir() {
   535→			// Legacy: skip subdirectories (only used by old Avro format).
   536→			// These are ignored; their data is not loaded.
   537→			cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir)
   538→			continue
   539→		}
   540→
   541→		name := e.Name()
   542→		if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") {
   543→			allFiles = append(allFiles, e)
   544→		} else if name == "current.wal" {
   545→			walEntry = e
   546→		}
   547→		// Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes).
   548→	}
   549→
   550→	files, err := findFiles(allFiles, from, true)
   551→	if err != nil {
   552→		return filesLoaded, err
   553→	}
   554→
   555→	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
   556→		".json": l.loadJSONFile,
   557→		".bin":  l.loadBinaryFile,
   558→	}
   559→
   560→	for _, filename := range files {
   561→		ext := filepath.Ext(filename)
   562→		loader := loaders[ext]
   563→		if loader == nil {
   564→			cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename)
   565→			continue
   566→		}
   567→
   568→		err := func() error {
   569→			f, err := os.Open(path.Join(dir, filename))
   570→			if err != nil {
   571→				return err
   572→			}
   573→			defer f.Close()
   574→			return loader(m, f, from)
   575→		}()
   576→		if err != nil {
   577→			return filesLoaded, err
   578→		}
   579→		filesLoaded++
   580→	}
   581→
   582→	// Replay WAL after all snapshot files so it fills in data since the last snapshot.
   583→	if walEntry != nil {
   584→		err := func() error {
   585→			f, err := os.Open(path.Join(dir, walEntry.Name()))
   586→			if err != nil {
   587→				return err
   588→			}
   589→			defer f.Close()
   590→			return l.loadWALFile(m, f, from)
   591→		}()
   592→		if err != nil {
   593→			// WAL errors are non-fatal: the snapshot already loaded the bulk of data.
   594→			cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err)
   595→		} else {
   596→			filesLoaded++
   597→		}
   598→	}
   599→
   600→	return filesLoaded, nil
   601→}
   602→
   603→// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename.
   604→// Supports ".json" (format: "<ts>.json") and ".bin" (format: "<ts>.bin").
   605→func parseTimestampFromFilename(name string) (int64, error) {
   606→	switch {
   607→	case strings.HasSuffix(name, ".json"):
   608→		return strconv.ParseInt(name[:len(name)-5], 10, 64)
   609→	case strings.HasSuffix(name, ".bin"):
   610→		return strconv.ParseInt(name[:len(name)-4], 10, 64)
   611→	default:
   612→		return 0, fmt.Errorf("unknown checkpoint extension for file %q", name)
   613→	}
   614→}
   615→
   616→// findFiles returns filenames from direntries whose timestamps satisfy the filter.
   617→// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the
   618→// last file before t if t falls between two files).
   619→func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {
   620→	nums := map[string]int64{}
   621→	for _, e := range direntries {
   622→		name := e.Name()
   623→		if !strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".bin") {
   624→			continue
   625→		}
   626→
   627→		ts, err := parseTimestampFromFilename(name)
   628→		if err != nil {
   629→			return nil, err
   630→		}
   631→		nums[name] = ts
   632→	}
   633→
   634→	sort.Slice(direntries, func(i, j int) bool {
   635→		a, b := direntries[i], direntries[j]
   636→		return nums[a.Name()] < nums[b.Name()]
   637→	})
   638→
   639→	if len(nums) == 0 {
   640→		return nil, nil
   641→	}
   642→
   643→	filenames := make([]string, 0)
   644→
   645→	for i, e := range direntries {
   646→		ts1 := nums[e.Name()]
   647→
   648→		if findMoreRecentFiles && t <= ts1 {
   649→			filenames = append(filenames, e.Name())
   650→		} else if !findMoreRecentFiles && ts1 <= t && ts1 != 0 {
   651→			filenames = append(filenames, e.Name())
   652→		}
   653→
   654→		if i == len(direntries)-1 {
   655→			continue
   656→		}
   657→
   658→		enext := direntries[i+1]
   659→		ts2 := nums[enext.Name()]
   660→
   661→		if findMoreRecentFiles {
   662→			if ts1 < t && t < ts2 {
   663→				filenames = append(filenames, e.Name())
   664→			}
   665→		}
   666→	}
   667→
   668→	return filenames, nil
   669→}
   670→


Grep

type Level struct|type buffer struct|type walMessage


pkg/metricstore/level.go:62:type Level struct {
pkg/metricstore/buffer.go:168:type buffer struct {


Bash

find pkg/metricstore -name "*.go" -type f | head -20


pkg/metricstore/query.go
pkg/metricstore/config.go
pkg/metricstore/walCheckpoint.go
pkg/metricstore/checkpoint.go
pkg/metricstore/parquetArchive.go
pkg/metricstore/healthcheck.go
pkg/metricstore/scopequery.go
pkg/metricstore/archive.go
pkg/metricstore/metricstore.go
pkg/metricstore/stats.go
pkg/metricstore/buffer.go
pkg/metricstore/debug.go
pkg/metricstore/level.go
pkg/metricstore/api.go
pkg/metricstore/lineprotocol.go
pkg/metricstore/metricstore_test.go
pkg/metricstore/parquetArchive_test.go
pkg/metricstore/configSchema.go
pkg/metricstore/scopequery_test.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides level.go: Hierarchical tree structure for metric storage.
     7→//
     8→// # Level Architecture
     9→//
    10→// The Level type forms a tree structure where each node represents a level in the
    11→// ClusterCockpit hierarchy: cluster → host → socket → core → hwthread, with special
    12→// nodes for memory domains and accelerators.
    13→//
    14→// Structure:
    15→//
    16→//	Root Level (cluster="emmy")
    17→//	├─ Level (host="node001")
    18→//	│  ├─ Level (socket="0")
    19→//	│  │  ├─ Level (core="0") [stores cpu0 metrics]
    20→//	│  │  └─ Level (core="1") [stores cpu1 metrics]
    21→//	│  └─ Level (socket="1")
    22→//	│     └─ ...
    23→//	└─ Level (host="node002")
    24→//	   └─ ...
    25→//
    26→// Each Level can:
    27→//   - Hold data (metrics slice of buffer pointers)
    28→//   - Have child nodes (children map[string]*Level)
    29→//   - Both simultaneously (inner nodes can store aggregated metrics)
    30→//
    31→// # Selector Paths
    32→//
    33→// Selectors are hierarchical paths: []string{"cluster", "host", "component"}.
    34→// Example: []string{"emmy", "node001", "cpu0"} navigates to the cpu0 core level.
    35→//
    36→// # Concurrency
    37→//
    38→// RWMutex protects children map and metrics slice. Read-heavy workload (metric reads)
    39→// uses RLock. Writes (new levels, buffer updates) use Lock. Double-checked locking
    40→// prevents races during level creation.
    41→package metricstore
    42→
    43→import (
    44→	"sync"
    45→	"time"
    46→	"unsafe"
    47→
    48→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    49→	"github.com/ClusterCockpit/cc-lib/v2/util"
    50→)
    51→
    52→// Level represents a node in the hierarchical metric storage tree.
    53→//
    54→// Can be both a leaf or inner node. Inner nodes hold data in 'metrics' for aggregated
    55→// values (e.g., socket-level metrics derived from core-level data). Named "Level"
    56→// instead of "node" to avoid confusion with cluster nodes (hosts).
    57→//
    58→// Fields:
    59→//   - children: Map of child level names to Level pointers (e.g., "cpu0" → Level)
    60→//   - metrics:  Slice of buffer pointers (one per metric, indexed by MetricConfig.offset)
    61→//   - lock:     RWMutex for concurrent access (read-heavy, write-rare)
    62→type Level struct {
    63→	children map[string]*Level
    64→	metrics  []*buffer
    65→	lock     sync.RWMutex
    66→}
    67→
    68→// findLevelOrCreate navigates to or creates the level specified by selector.
    69→//
    70→// Recursively descends the tree, creating missing levels as needed. Uses double-checked
    71→// locking: RLock first (fast path), then Lock if creation needed (slow path), then
    72→// re-check after acquiring Lock to handle races.
    73→//
    74→// Example selector: []string{"emmy", "node001", "cpu0"}
    75→// Navigates: root → emmy → node001 → cpu0, creating levels as needed.
    76→//
    77→// Parameters:
    78→//   - selector: Hierarchical path (consumed recursively, decreasing depth)
    79→//   - nMetrics: Number of metric slots to allocate in new levels
    80→//
    81→// Returns:
    82→//   - *Level: The target level (existing or newly created)
    83→//
    84→// Note: sync.Map may improve performance for high-concurrency writes, but current
    85→// approach suffices for read-heavy workload.
    86→func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
    87→	if len(selector) == 0 {
    88→		return l
    89→	}
    90→
    91→	// Allow concurrent reads:
    92→	l.lock.RLock()
    93→	var child *Level
    94→	var ok bool
    95→	if l.children == nil {
    96→		// Children map needs to be created...
    97→		l.lock.RUnlock()
    98→	} else {
    99→		child, ok = l.children[selector[0]]
   100→		l.lock.RUnlock()
   101→		if ok {
   102→			return child.findLevelOrCreate(selector[1:], nMetrics)
   103→		}
   104→	}
   105→
   106→	// The level does not exist, take write lock for unique access:
   107→	l.lock.Lock()
   108→	// While this thread waited for the write lock, another thread
   109→	// could have created the child node.
   110→	if l.children != nil {
   111→		child, ok = l.children[selector[0]]
   112→		if ok {
   113→			l.lock.Unlock()
   114→			return child.findLevelOrCreate(selector[1:], nMetrics)
   115→		}
   116→	}
   117→
   118→	child = &Level{
   119→		metrics:  make([]*buffer, nMetrics),
   120→		children: nil,
   121→	}
   122→
   123→	if l.children != nil {
   124→		l.children[selector[0]] = child
   125→	} else {
   126→		l.children = map[string]*Level{selector[0]: child}
   127→	}
   128→	l.lock.Unlock()
   129→	return child.findLevelOrCreate(selector[1:], nMetrics)
   130→}
   131→
   132→// collectPaths gathers all selector paths at the specified depth in the tree.
   133→//
   134→// Recursively traverses children, collecting paths when currentDepth+1 == targetDepth.
   135→// Each path is a selector that can be used with findLevel() or findBuffers().
   136→//
   137→// Explicitly copies slices to avoid shared underlying arrays between siblings, preventing
   138→// unintended mutations.
   139→//
   140→// Parameters:
   141→//   - currentDepth: Depth of current level (0 = root)
   142→//   - targetDepth:  Depth to collect paths from
   143→//   - currentPath:  Path accumulated so far
   144→//   - results:      Output slice (appended to)
   145→//
   146→// Example: collectPaths(0, 2, []string{}, &results) collects all 2-level paths
   147→// like []string{"emmy", "node001"}, []string{"emmy", "node002"}, etc.
   148→func (l *Level) collectPaths(currentDepth, targetDepth int, currentPath []string, results *[][]string) {
   149→	l.lock.RLock()
   150→	defer l.lock.RUnlock()


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides buffer.go: Time-series data buffer implementation.
     7→//
     8→// # Buffer Architecture
     9→//
    10→// Each metric at each hierarchical level (cluster/host/cpu/etc.) uses a linked-list
    11→// chain of fixed-size buffers to store time-series data. This design:
    12→//
    13→//   - Avoids reallocation/copying when growing (new links added instead)
    14→//   - Enables efficient pooling (buffers returned to sync.Pool)
    15→//   - Supports traversal back in time (via prev pointers)
    16→//   - Maintains temporal ordering (newer data in later buffers)
    17→//
    18→// # Buffer Chain Example
    19→//
    20→//	[oldest buffer] <- prev -- [older] <- prev -- [newest buffer (head)]
    21→//	  start=1000               start=1512           start=2024
    22→//	  data=[v0...v511]         data=[v0...v511]     data=[v0...v42]
    23→//
    24→// When the head buffer reaches capacity (BufferCap = 512), a new buffer becomes
    25→// the new head and the old head is linked via prev.
    26→//
    27→// # Pooling Strategy
    28→//
    29→// sync.Pool reduces GC pressure for the common case (BufferCap-sized allocations).
    30→// Non-standard capacity buffers are not pooled (e.g., from checkpoint deserialization).
    31→//
    32→// # Time Alignment
    33→//
    34→// Timestamps are aligned to measurement frequency intervals:
    35→//
    36→//	index = (timestamp - buffer.start) / buffer.frequency
    37→//	actualTime = buffer.start + (frequency / 2) + (index * frequency)
    38→//
    39→// Missing data points are represented as NaN values. The read() function performs
    40→// linear interpolation where possible.
    41→package metricstore
    42→
    43→import (
    44→	"errors"
    45→	"sync"
    46→	"time"
    47→
    48→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    49→)
    50→
    51→// BufferCap is the default buffer capacity.
    52→// buffer.data will only ever grow up to its capacity and a new link
    53→// in the buffer chain will be created if needed so that no copying
    54→// of data or reallocation needs to happen on writes.
    55→const BufferCap int = DefaultBufferCapacity
    56→
    57→// BufferPool is the global instance.
    58→// It is initialized immediately when the package loads.
    59→var bufferPool = NewPersistentBufferPool()
    60→
    61→type PersistentBufferPool struct {
    62→	pool []*buffer
    63→	mu   sync.Mutex
    64→}
    65→
    66→// NewPersistentBufferPool creates a dynamic pool for buffers.
    67→func NewPersistentBufferPool() *PersistentBufferPool {
    68→	return &PersistentBufferPool{
    69→		pool: make([]*buffer, 0),
    70→	}
    71→}
    72→
    73→func (p *PersistentBufferPool) Get() *buffer {
    74→	p.mu.Lock()
    75→	defer p.mu.Unlock()
    76→
    77→	n := len(p.pool)
    78→	if n == 0 {
    79→		// Pool is empty, allocate a new one
    80→		return &buffer{
    81→			data: make([]schema.Float, 0, BufferCap),
    82→		}
    83→	}
    84→
    85→	// Reuse existing buffer from the pool
    86→	b := p.pool[n-1]
    87→	p.pool[n-1] = nil // Avoid memory leak
    88→	p.pool = p.pool[:n-1]
    89→	return b
    90→}
    91→
    92→// Put returns b to the pool. The caller must set b.lastUsed = time.Now().Unix()
    93→// before calling Put so that Clean() can evict idle entries correctly.
    94→func (p *PersistentBufferPool) Put(b *buffer) {
    95→	// Reset the buffer before putting it back
    96→	b.data = b.data[:0]
    97→
    98→	p.mu.Lock()
    99→	defer p.mu.Unlock()
   100→
   101→	p.pool = append(p.pool, b)
   102→}
   103→
   104→// GetSize returns the exact number of buffers currently sitting in the pool.
   105→func (p *PersistentBufferPool) GetSize() int {
   106→	p.mu.Lock()
   107→	defer p.mu.Unlock()
   108→	return len(p.pool)
   109→}
   110→
   111→// Clear drains all buffers currently in the pool, allowing the GC to collect them.
   112→func (p *PersistentBufferPool) Clear() {
   113→	p.mu.Lock()
   114→	defer p.mu.Unlock()
   115→	for i := range p.pool {
   116→		p.pool[i] = nil
   117→	}
   118→	p.pool = p.pool[:0]
   119→}
   120→
   121→// Clean removes buffers from the pool that haven't been used in the given duration.
   122→// It uses a simple LRU approach based on the lastUsed timestamp.
   123→func (p *PersistentBufferPool) Clean(threshold int64) {
   124→	p.mu.Lock()
   125→	defer p.mu.Unlock()
   126→
   127→	// Filter in place, retaining only buffers returned to the pool recently enough.
   128→	active := p.pool[:0]
   129→	for _, b := range p.pool {
   130→		if b.lastUsed >= threshold {
   131→			active = append(active, b)
   132→		}
   133→	}
   134→
   135→	// Nullify the rest to prevent memory leaks
   136→	for i := len(active); i < len(p.pool); i++ {
   137→		p.pool[i] = nil
   138→	}
   139→
   140→	p.pool = active
   141→}
   142→
   143→var (
   144→	// ErrNoData indicates no time-series data exists for the requested metric/level.
   145→	ErrNoData error = errors.New("[METRICSTORE]> no data for this metric/level")
   146→
   147→	// ErrDataDoesNotAlign indicates that aggregated data from child scopes
   148→	// does not align with the parent scope's expected timestamps/intervals.
   149→	ErrDataDoesNotAlign error = errors.New("[METRICSTORE]> data from lower granularities does not align")
   150→)
   151→
   152→// buffer stores time-series data for a single metric at a specific hierarchical level.
   153→//
   154→// Buffers form doubly-linked chains ordered by time. When capacity is reached,
   155→// a new buffer becomes the head and the old head is linked via prev/next.
   156→//
   157→// Fields:
   158→//   - prev:      Link to older buffer in the chain (nil if this is oldest)
   159→//   - next:      Link to newer buffer in the chain (nil if this is newest/head)
   160→//   - data:      Time-series values (schema.Float supports NaN for missing data)
   161→//   - frequency: Measurement interval in seconds
   162→//   - start:     Start timestamp (adjusted by -frequency/2 for alignment)
   163→//   - archived:  True if data has been persisted to disk archive
   164→//   - closed:    True if buffer is no longer accepting writes
   165→//
   166→// Index calculation: index = (timestamp - start) / frequency
   167→// Actual data timestamp: start + (frequency / 2) + (index * frequency)
   168→type buffer struct {
   169→	prev      *buffer
   170→	next      *buffer
   171→	data      []schema.Float
   172→	frequency int64
   173→	start     int64
   174→	archived  bool
   175→	closed    bool
   176→	lastUsed  int64
   177→}
   178→
   179→func newBuffer(ts, freq int64) *buffer {
   180→	b := bufferPool.Get()
   181→	b.frequency = freq
   182→	b.start = ts - (freq / 2)
   183→	b.prev = nil
   184→	b.next = nil
   185→	b.archived = false
   186→	b.closed = false
   187→	b.data = b.data[:0]
   188→	return b
   189→}
   190→
   191→// write appends a timestamped value to the buffer chain.
   192→//
   193→// Returns the head buffer (which may be newly created if capacity was reached).
   194→// Timestamps older than the buffer's start are rejected. If the calculated index
   195→// exceeds capacity, a new buffer is allocated and linked as the new head.
   196→//
   197→// Missing timestamps are automatically filled with NaN values to maintain alignment.
   198→// Overwrites are allowed if the index is already within the existing data slice.
   199→//
   200→// Parameters:
   201→//   - ts:    Unix timestamp in seconds
   202→//   - value: Metric value (can be schema.NaN for missing data)
   203→//
   204→// Returns:
   205→//   - *buffer: The new head buffer (same as b if no new buffer created)
   206→//   - error:   Non-nil if timestamp is before buffer start
   207→func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
   208→	if ts < b.start {
   209→		return nil, errors.New("[METRICSTORE]> cannot write value to buffer from past")
   210→	}
   211→
   212→	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
   213→	idx := int((ts - b.start) / b.frequency)
   214→	if idx >= cap(b.data) {
   215→		newbuf := newBuffer(ts, b.frequency)
   216→		newbuf.prev = b
   217→		b.next = newbuf
   218→		b = newbuf
   219→		idx = 0
   220→	}
   221→
   222→	// Overwriting value or writing value from past
   223→	if idx < len(b.data) {
   224→		b.data[idx] = value
   225→		return b, nil
   226→	}
   227→
   228→	// Fill up unwritten slots with NaN
   229→	for i := len(b.data); i < idx; i++ {
   230→		b.data = append(b.data, schema.NaN)
   231→	}
   232→
   233→	b.data = append(b.data, value)
   234→	return b, nil
   235→}
   236→
   237→func (b *buffer) end() int64 {
   238→	return b.firstWrite() + int64(len(b.data))*b.frequency
   239→}
   240→
   241→func (b *buffer) firstWrite() int64 {
   242→	return b.start + (b.frequency / 2)
   243→}
   244→
   245→// read retrieves time-series data from the buffer chain for the specified time range.
   246→//
   247→// Traverses the buffer chain backwards (via prev links) if 'from' precedes the current
   248→// buffer's start. Missing data points are represented as NaN. Values are accumulated
   249→// into the provided 'data' slice (using +=, so caller must zero-initialize if needed).
   250→//