     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides buffer.go: Time-series data buffer implementation.
     7→//
     8→// # Buffer Architecture
     9→//
    10→// Each metric at each hierarchical level (cluster/host/cpu/etc.) uses a linked-list
    11→// chain of fixed-size buffers to store time-series data. This design:
    12→//
    13→//   - Avoids reallocation/copying when growing (new links added instead)
    14→//   - Enables efficient pooling (buffers returned to sync.Pool)
    15→//   - Supports traversal back in time (via prev pointers)
    16→//   - Maintains temporal ordering (newer data in later buffers)
    17→//
    18→// # Buffer Chain Example
    19→//
    20→//	[oldest buffer] <- prev -- [older] <- prev -- [newest buffer (head)]
    21→//	  start=1000               start=1512           start=2024
    22→//	  data=[v0...v511]         data=[v0...v511]     data=[v0...v42]
    23→//
    24→// When the head buffer reaches capacity (BufferCap = 512), a new buffer becomes
    25→// the new head and the old head is linked via prev.
    26→//
    27→// # Pooling Strategy
    28→//
    29→// sync.Pool reduces GC pressure for the common case (BufferCap-sized allocations).
    30→// Non-standard capacity buffers are not pooled (e.g., from checkpoint deserialization).
    31→//
    32→// # Time Alignment
    33→//
    34→// Timestamps are aligned to measurement frequency intervals:
    35→//
    36→//	index = (timestamp - buffer.start) / buffer.frequency
    37→//	actualTime = buffer.start + (frequency / 2) + (index * frequency)
    38→//
    39→// Missing data points are represented as NaN values. The read() function performs
    40→// linear interpolation where possible.
    41→package metricstore
    42→
    43→import (
    44→	"errors"
    45→	"sync"
    46→	"time"
    47→
    48→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    49→)
    50→
    51→// BufferCap is the default buffer capacity.
    52→// buffer.data will only ever grow up to its capacity and a new link
    53→// in the buffer chain will be created if needed so that no copying
    54→// of data or reallocation needs to happen on writes.
    55→const BufferCap int = DefaultBufferCapacity
    56→
    57→// BufferPool is the global instance.
    58→// It is initialized immediately when the package loads.
    59→var bufferPool = NewPersistentBufferPool()
    60→
    61→type PersistentBufferPool struct {
    62→	pool []*buffer
    63→	mu   sync.Mutex
    64→}
    65→
    66→// NewPersistentBufferPool creates a dynamic pool for buffers.
    67→func NewPersistentBufferPool() *PersistentBufferPool {
    68→	return &PersistentBufferPool{
    69→		pool: make([]*buffer, 0),
    70→	}
    71→}
    72→
    73→func (p *PersistentBufferPool) Get() *buffer {
    74→	p.mu.Lock()
    75→	defer p.mu.Unlock()
    76→
    77→	n := len(p.pool)
    78→	if n == 0 {
    79→		// Pool is empty, allocate a new one
    80→		return &buffer{
    81→			data: make([]schema.Float, 0, BufferCap),
    82→		}
    83→	}
    84→
    85→	// Reuse existing buffer from the pool
    86→	b := p.pool[n-1]
    87→	p.pool[n-1] = nil // Avoid memory leak
    88→	p.pool = p.pool[:n-1]
    89→	return b
    90→}
    91→
    92→// Put returns b to the pool. The caller must set b.lastUsed = time.Now().Unix()
    93→// before calling Put so that Clean() can evict idle entries correctly.
    94→func (p *PersistentBufferPool) Put(b *buffer) {
    95→	// Reset the buffer before putting it back
    96→	b.data = b.data[:0]
    97→
    98→	p.mu.Lock()
    99→	defer p.mu.Unlock()
   100→
   101→	p.pool = append(p.pool, b)
   102→}
   103→
   104→// GetSize returns the exact number of buffers currently sitting in the pool.
   105→func (p *PersistentBufferPool) GetSize() int {
   106→	p.mu.Lock()
   107→	defer p.mu.Unlock()
   108→	return len(p.pool)
   109→}
   110→
   111→// Clear drains all buffers currently in the pool, allowing the GC to collect them.
   112→func (p *PersistentBufferPool) Clear() {
   113→	p.mu.Lock()
   114→	defer p.mu.Unlock()
   115→	for i := range p.pool {
   116→		p.pool[i] = nil
   117→	}
   118→	p.pool = p.pool[:0]
   119→}
   120→
   121→// Clean removes buffers from the pool that haven't been used in the given duration.
   122→// It uses a simple LRU approach based on the lastUsed timestamp.
   123→func (p *PersistentBufferPool) Clean(threshold int64) {
   124→	p.mu.Lock()
   125→	defer p.mu.Unlock()
   126→
   127→	// Filter in place, retaining only buffers returned to the pool recently enough.
   128→	active := p.pool[:0]
   129→	for _, b := range p.pool {
   130→		if b.lastUsed >= threshold {
   131→			active = append(active, b)
   132→		}
   133→	}
   134→
   135→	// Nullify the rest to prevent memory leaks
   136→	for i := len(active); i < len(p.pool); i++ {
   137→		p.pool[i] = nil
   138→	}
   139→
   140→	p.pool = active
   141→}
   142→
   143→var (
   144→	// ErrNoData indicates no time-series data exists for the requested metric/level.
   145→	ErrNoData error = errors.New("[METRICSTORE]> no data for this metric/level")
   146→
   147→	// ErrDataDoesNotAlign indicates that aggregated data from child scopes
   148→	// does not align with the parent scope's expected timestamps/intervals.
   149→	ErrDataDoesNotAlignMissingFront    error = errors.New("[METRICSTORE]> data from lower granularities does not align (missing data prior to start of the buffers)")
   150→	ErrDataDoesNotAlignMissingBack     error = errors.New("[METRICSTORE]> data from lower granularities does not align (missing data after the end of the buffers)")
   151→	ErrDataDoesNotAlignDataLenMismatch error = errors.New("[METRICSTORE]> data from lower granularities does not align (collected data length is different than expected data length)")
   152→)
   153→
   154→// buffer stores time-series data for a single metric at a specific hierarchical level.
   155→//
   156→// Buffers form doubly-linked chains ordered by time. When capacity is reached,
   157→// a new buffer becomes the head and the old head is linked via prev/next.
   158→//
   159→// Fields:
   160→//   - prev:      Link to older buffer in the chain (nil if this is oldest)
   161→//   - next:      Link to newer buffer in the chain (nil if this is newest/head)
   162→//   - data:      Time-series values (schema.Float supports NaN for missing data)
   163→//   - frequency: Measurement interval in seconds
   164→//   - start:     Start timestamp (adjusted by -frequency/2 for alignment)
   165→//   - archived:  True if data has been persisted to disk archive
   166→//   - closed:    True if buffer is no longer accepting writes
   167→//
   168→// Index calculation: index = (timestamp - start) / frequency
   169→// Actual data timestamp: start + (frequency / 2) + (index * frequency)
   170→type buffer struct {
   171→	prev      *buffer
   172→	next      *buffer
   173→	data      []schema.Float
   174→	frequency int64
   175→	start     int64
   176→	archived  bool
   177→	closed    bool
   178→	lastUsed  int64
   179→}
   180→
   181→func newBuffer(ts, freq int64) *buffer {
   182→	b := bufferPool.Get()
   183→	b.frequency = freq
   184→	b.start = ts - (freq / 2)
   185→	b.prev = nil
   186→	b.next = nil
   187→	b.archived = false
   188→	b.closed = false
   189→	b.data = b.data[:0]
   190→	return b
   191→}
   192→
   193→// write appends a timestamped value to the buffer chain.
   194→//
   195→// Returns the head buffer (which may be newly created if capacity was reached).
   196→// Timestamps older than the buffer's start are rejected. If the calculated index
   197→// exceeds capacity, a new buffer is allocated and linked as the new head.
   198→//
   199→// Missing timestamps are automatically filled with NaN values to maintain alignment.
   200→// Overwrites are allowed if the index is already within the existing data slice.
   201→//
   202→// Parameters:
   203→//   - ts:    Unix timestamp in seconds
   204→//   - value: Metric value (can be schema.NaN for missing data)
   205→//
   206→// Returns:
   207→//   - *buffer: The new head buffer (same as b if no new buffer created)
   208→//   - error:   Non-nil if timestamp is before buffer start
   209→func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
   210→	if ts < b.start {
   211→		return nil, errors.New("[METRICSTORE]> cannot write value to buffer from past")
   212→	}
   213→
   214→	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
   215→	idx := int((ts - b.start) / b.frequency)
   216→	if idx >= cap(b.data) {
   217→		newbuf := newBuffer(ts, b.frequency)
   218→		newbuf.prev = b
   219→		b.next = newbuf
   220→		b = newbuf
   221→		idx = 0
   222→	}
   223→
   224→	// Overwriting value or writing value from past
   225→	if idx < len(b.data) {
   226→		b.data[idx] = value
   227→		return b, nil
   228→	}
   229→
   230→	// Fill up unwritten slots with NaN
   231→	for i := len(b.data); i < idx; i++ {
   232→		b.data = append(b.data, schema.NaN)
   233→	}
   234→
   235→	b.data = append(b.data, value)
   236→	return b, nil
   237→}
   238→
   239→func (b *buffer) end() int64 {
   240→	return b.firstWrite() + int64(len(b.data))*b.frequency
   241→}
   242→
   243→func (b *buffer) firstWrite() int64 {
   244→	return b.start + (b.frequency / 2)
   245→}
   246→
   247→// read retrieves time-series data from the buffer chain for the specified time range.
   248→//
   249→// Traverses the buffer chain backwards (via prev links) if 'from' precedes the current
   250→// buffer's start. Missing data points are represented as NaN. Values are accumulated
   251→// into the provided 'data' slice (using +=, so caller must zero-initialize if needed).
   252→//
   253→// The function adjusts the actual time range returned if data is unavailable at the
   254→// boundaries (returned via adjusted from/to timestamps).
   255→//
   256→// Parameters:
   257→//   - from: Start timestamp (Unix seconds)
   258→//   - to:   End timestamp (Unix seconds, exclusive)
   259→//   - data: Pre-allocated slice to accumulate results (must be large enough)
   260→//
   261→// Returns:
   262→//   - []schema.Float: Slice of data (may be shorter than input 'data' slice)
   263→//   - int64:          Actual start timestamp with available data
   264→//   - int64:          Actual end timestamp (exclusive)
   265→//   - error:          Non-nil on failure
   266→//
   267→// Panics if 'data' slice is too small to hold all values in [from, to).
   268→func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) {
   269→	// Walk back to the buffer that covers 'from', adjusting if we hit the oldest.
   270→	for from < b.firstWrite() {
   271→		if b.prev == nil {
   272→			from = b.firstWrite()
   273→			break
   274→		}
   275→		b = b.prev
   276→	}
   277→
   278→	i := 0
   279→	t := from
   280→	for ; t < to; t += b.frequency {
   281→		idx := int((t - b.start) / b.frequency)
   282→		if idx >= cap(b.data) {
   283→			if b.next == nil {
   284→				break
   285→			}
   286→			b = b.next
   287→			// Recalculate idx in the new buffer; a gap between buffers may exist.
   288→			idx = int((t - b.start) / b.frequency)
   289→		}
   290→
   291→		if idx >= len(b.data) {
   292→			if b.next == nil || to <= b.next.start {
   293→				break
   294→			}
   295→			data[i] += schema.NaN // NaN + anything = NaN; propagates missing data
   296→		} else if t < b.start {
   297→			data[i] += schema.NaN // gap before this buffer's first write
   298→		} else {
   299→			data[i] += b.data[idx]
   300→		}
   301→		i++
   302→	}
   303→
   304→	return data[:i], from, t, nil
   305→}
   306→
   307→// free removes buffers older than the specified timestamp from the chain.
   308→//
   309→// Recursively traverses backwards (via prev) and unlinks buffers whose end time
   310→// is before the retention threshold. Freed buffers are returned to the pool if
   311→// they have the standard capacity (BufferCap).
   312→//
   313→// Parameters:
   314→//   - t: Retention threshold timestamp (Unix seconds)
   315→//
   316→// Returns:
   317→//   - delme: True if the current buffer itself should be deleted by caller
   318→//   - n:     Number of buffers freed in this subtree
   319→func (b *buffer) free(t int64) (delme bool, n int) {
   320→	if b.prev != nil {
   321→		delme, m := b.prev.free(t)
   322→		n += m
   323→		if delme {
   324→			b.prev.next = nil
   325→			if cap(b.prev.data) != BufferCap {
   326→				b.prev.data = make([]schema.Float, 0, BufferCap)
   327→			}
   328→			b.prev.lastUsed = time.Now().Unix()
   329→			bufferPool.Put(b.prev)
   330→			b.prev = nil
   331→		}
   332→	}
   333→
   334→	end := b.end()
   335→	if end < t {
   336→		return true, n + 1
   337→	}
   338→
   339→	return false, n
   340→}
   341→
   342→// forceFreeOldest recursively finds the end of the linked list (the oldest buffer)
   343→// and removes it.
   344→// Returns:
   345→//
   346→//	delme: true if 'b' itself is the oldest and should be removed by the caller
   347→//	n:     the number of buffers freed (will be 1 or 0)
   348→func (b *buffer) forceFreeOldest() (delme bool, n int) {
   349→	// If there is a previous buffer, recurse down to find the oldest
   350→	if b.prev != nil {
   351→		delPrev, freed := b.prev.forceFreeOldest()
   352→
   353→		// If the previous buffer signals it should be deleted:
   354→		if delPrev {
   355→			b.prev.next = nil
   356→			if cap(b.prev.data) != BufferCap {
   357→				b.prev.data = make([]schema.Float, 0, BufferCap)
   358→			}
   359→			b.prev.lastUsed = time.Now().Unix()
   360→			bufferPool.Put(b.prev)
   361→			b.prev = nil
   362→		}
   363→		return false, freed
   364→	}
   365→
   366→	// If b.prev is nil, THIS buffer is the oldest.
   367→	// We return true so the parent (or the Level loop) knows to delete reference to 'b'.
   368→	return true, 1
   369→}
   370→
   371→// iterFromTo invokes callback on every buffer in the chain that overlaps [from, to].
   372→//
   373→// Traverses backwards (via prev) first, then processes current buffer if it overlaps
   374→// the time range. Used for checkpoint/archive operations that need to serialize buffers
   375→// within a specific time window.
   376→//
   377→// Parameters:
   378→//   - from:     Start timestamp (Unix seconds, inclusive)
   379→//   - to:       End timestamp (Unix seconds, inclusive)
   380→//   - callback: Function to invoke on each overlapping buffer
   381→//
   382→// Returns:
   383→//   - error: First error returned by callback, or nil if all succeeded
   384→func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error {
   385→	if b == nil {
   386→		return nil
   387→	}
   388→
   389→	// Collect overlapping buffers walking backwards (newest → oldest).
   390→	var matching []*buffer
   391→	for cur := b; cur != nil; cur = cur.prev {
   392→		if from <= cur.end() && cur.start <= to {
   393→			matching = append(matching, cur)
   394→		}
   395→	}
   396→
   397→	// Invoke callback in chronological order (oldest → newest).
   398→	for i := len(matching) - 1; i >= 0; i-- {
   399→		if err := callback(matching[i]); err != nil {
   400→			return err
   401→		}
   402→	}
   403→	return nil
   404→}
   405→
   406→func (b *buffer) count() int64 {
   407→	var res int64
   408→	for ; b != nil; b = b.prev {
   409→		res += int64(len(b.data))
   410→	}
   411→	return res
   412→}
   413→
   414→// bufferCount returns the number of buffer nodes in the linked list.
   415→func (b *buffer) bufferCount() int64 {
   416→	var n int64
   417→	for ; b != nil; b = b.prev {
   418→		n++
   419→	}
   420→	return n
   421→}
   422→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
     7→//
     8→// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
     9→// per host, eliminating the extra memory overhead of the AvroStore and providing
    10→// truly continuous (per-write) crash safety.
    11→//
    12→// # Architecture
    13→//
    14→//	Metric write (DecodeLine)
    15→//	      │
    16→//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
    17→//	      │
    18→//	      └─► WALMessages channel
    19→//	               │
    20→//	               ▼
    21→//	        WALStaging goroutine
    22→//	               │
    23→//	               ▼
    24→//	        checkpoints/cluster/host/current.wal  (append-only, binary)
    25→//
    26→//	Periodic checkpoint (Checkpointing goroutine):
    27→//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
    28→//	  2. Signal WALStaging to truncate current.wal per host
    29→//
    30→//	On restart (FromCheckpoint):
    31→//	  1. Load most recent <timestamp>.bin snapshot
    32→//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
    33→//
    34→// # WAL Record Format
    35→//
    36→//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
    37→//
    38→//	payload:
    39→//	  [8B timestamp int64]
    40→//	  [2B metric_name_len uint16][N metric name bytes]
    41→//	  [1B selector_count uint8]
    42→//	  per selector: [1B selector_len uint8][M selector bytes]
    43→//	  [4B value float32 bits]
    44→//
    45→// # Binary Snapshot Format
    46→//
    47→//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
    48→//	Level tree (recursive):
    49→//	  [4B num_metrics uint32]
    50→//	  per metric:
    51→//	    [2B name_len uint16][N name bytes]
    52→//	    [8B frequency int64][8B start int64]
    53→//	    [4B num_values uint32][num_values × 4B float32]
    54→//	  [4B num_children uint32]
    55→//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
    56→package metricstore
    57→
    58→import (
    59→	"bufio"
    60→	"context"
    61→	"encoding/binary"
    62→	"fmt"
    63→	"hash/crc32"
    64→	"hash/fnv"
    65→	"io"
    66→	"math"
    67→	"os"
    68→	"path"
    69→	"strings"
    70→	"sync"
    71→	"sync/atomic"
    72→	"time"
    73→
    74→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    75→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    76→)
    77→
    78→// Magic numbers for binary formats.
    79→const (
    80→	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
    81→	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
    82→	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
    83→)
    84→
    85→// walShardChs holds per-shard channels for WAL messages.
    86→// Initialized by WALStaging; nil when WAL is not active.
    87→var walShardChs []chan *WALMessage
    88→
    89→// walShardRotateChs holds per-shard channels for WAL rotation requests.
    90→var walShardRotateChs []chan walRotateReq
    91→
    92→// walNumShards stores the number of shards (set during WALStaging init).
    93→var walNumShards int
    94→
    95→// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
    96→var walStagingWg sync.WaitGroup
    97→
    98→// walShuttingDown is set before closing shard channels to prevent
    99→// SendWALMessage from sending on a closed channel (which panics in Go).
   100→var walShuttingDown atomic.Bool
   101→
   102→// walCheckpointActive is set during binary checkpoint writes.
   103→// While active, SendWALMessage skips sending (returns true) because the
   104→// snapshot captures all in-memory data, making WAL writes redundant.
   105→var walCheckpointActive atomic.Bool
   106→
   107→// WALMessage represents a single metric write to be appended to the WAL.
   108→// Cluster and Node are NOT stored in the WAL record (inferred from file path).
   109→type WALMessage struct {
   110→	MetricName string
   111→	Cluster    string
   112→	Node       string
   113→	Selector   []string
   114→	Value      schema.Float
   115→	Timestamp  int64
   116→}
   117→
   118→// walRotateReq requests WAL file rotation for a specific host directory.
   119→// The done channel is closed by the WAL goroutine when rotation is complete.
   120→type walRotateReq struct {
   121→	hostDir string
   122→	done    chan struct{}
   123→}
   124→
   125→// walFileState holds an open WAL file handle and buffered writer for one host directory.
   126→type walFileState struct {
   127→	f     *os.File
   128→	w     *bufio.Writer
   129→	dirty bool
   130→	size  int64 // approximate bytes written (tracked from open + writes)
   131→}
   132→
   133→// walFlushInterval controls how often dirty WAL files are flushed to disk.
   134→// Decoupling flushes from message processing lets the consumer run at memory
   135→// speed, amortizing syscall overhead across many writes.
   136→const walFlushInterval = 1 * time.Second
   137→
   138→// walShardIndex computes which shard a message belongs to based on cluster+node.
   139→// Uses FNV-1a hash for fast, well-distributed mapping.
   140→func walShardIndex(cluster, node string) int {
   141→	h := fnv.New32a()
   142→	h.Write([]byte(cluster))
   143→	h.Write([]byte{0})
   144→	h.Write([]byte(node))
   145→	return int(h.Sum32()) % walNumShards
   146→}
   147→
   148→// SendWALMessage routes a WAL message to the appropriate shard channel.
   149→// Returns false if the channel is full or shutdown is in progress.
   150→func SendWALMessage(msg *WALMessage) bool {
   151→	if walShardChs == nil || walShuttingDown.Load() {
   152→		return false
   153→	}
   154→	if walCheckpointActive.Load() {
   155→		return true // Data safe in memory; snapshot will capture it
   156→	}
   157→	shard := walShardIndex(msg.Cluster, msg.Node)
   158→	select {
   159→	case walShardChs[shard] <- msg:
   160→		return true
   161→	default:
   162→		return false
   163→	}
   164→}
   165→
   166→// WALStaging starts N sharded background goroutines that receive WALMessage items
   167→// and append binary WAL records to per-host current.wal files.
   168→// Each shard owns a subset of hosts (determined by hash of cluster+node),
   169→// parallelizing serialization and I/O while keeping per-host writes serial.
   170→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   171→	if Keys.Checkpoints.FileFormat == "json" {
   172→		return
   173→	}
   174→
   175→	walNumShards = max(Keys.NumWorkers, 1)
   176→	chBufSize := max(65536/walNumShards, 1024)
   177→
   178→	walShardChs = make([]chan *WALMessage, walNumShards)
   179→	walShardRotateChs = make([]chan walRotateReq, walNumShards)
   180→
   181→	for i := range walNumShards {
   182→		walShardChs[i] = make(chan *WALMessage, chBufSize)
   183→		walShardRotateChs[i] = make(chan walRotateReq, 64)
   184→	}
   185→
   186→	for i := range walNumShards {
   187→		msgCh := walShardChs[i]
   188→		rotateCh := walShardRotateChs[i]
   189→
   190→		walStagingWg.Add(1)
   191→		wg.Go(func() {
   192→			defer walStagingWg.Done()
   193→			hostFiles := make(map[string]*walFileState)
   194→
   195→			defer func() {
   196→				for _, ws := range hostFiles {
   197→					if ws.f != nil {
   198→						ws.w.Flush()
   199→						ws.f.Close()
   200→					}
   201→				}
   202→			}()
   203→
   204→			getOrOpenWAL := func(hostDir string) *walFileState {
   205→				ws, ok := hostFiles[hostDir]
   206→				if ok {
   207→					return ws
   208→				}
   209→
   210→				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   211→					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   212→					return nil
   213→				}
   214→
   215→				walPath := path.Join(hostDir, "current.wal")
   216→				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   217→				if err != nil {
   218→					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   219→					return nil
   220→				}
   221→
   222→				w := bufio.NewWriter(f)
   223→
   224→				// Write file header magic if file is new (empty).
   225→				info, err := f.Stat()
   226→				var fileSize int64
   227→				if err == nil {
   228→					fileSize = info.Size()
   229→				}
   230→				if err == nil && fileSize == 0 {
   231→					var hdr [4]byte
   232→					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   233→					if _, err := w.Write(hdr[:]); err != nil {
   234→						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   235→						f.Close()
   236→						return nil
   237→					}
   238→					fileSize = 4
   239→				}
   240→
   241→				ws = &walFileState{f: f, w: w, size: fileSize}
   242→				hostFiles[hostDir] = ws
   243→				return ws
   244→			}
   245→
   246→			processMsg := func(msg *WALMessage) {
   247→				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   248→				ws := getOrOpenWAL(hostDir)
   249→				if ws == nil {
   250→					return
   251→				}
   252→
   253→				// Enforce max WAL size: force-rotate before writing if limit is exceeded.
   254→				// The in-memory store still holds the data; only crash-recovery coverage is lost.
   255→				if maxSize := Keys.Checkpoints.MaxWALSize; maxSize > 0 && ws.size >= maxSize {
   256→					cclog.Warnf("[METRICSTORE]> WAL: force-rotating %s (size %d >= limit %d)",
   257→						hostDir, ws.size, maxSize)
   258→					ws.w.Flush()
   259→					ws.f.Close()
   260→					walPath := path.Join(hostDir, "current.wal")
   261→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   262→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   263→					}
   264→					delete(hostFiles, hostDir)
   265→					ws = getOrOpenWAL(hostDir)
   266→					if ws == nil {
   267→						return
   268→					}
   269→				}
   270→
   271→				n, err := writeWALRecordDirect(ws.w, msg)
   272→				if err != nil {
   273→					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   274→				}
   275→				ws.size += int64(n)
   276→				ws.dirty = true
   277→			}
   278→
   279→			processRotate := func(req walRotateReq) {
   280→				ws, ok := hostFiles[req.hostDir]
   281→				if ok && ws.f != nil {
   282→					ws.w.Flush()
   283→					ws.f.Close()
   284→					walPath := path.Join(req.hostDir, "current.wal")
   285→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   286→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   287→					}
   288→					delete(hostFiles, req.hostDir)
   289→				}
   290→				close(req.done)
   291→			}
   292→
   293→			flushDirty := func() {
   294→				for _, ws := range hostFiles {
   295→					if ws.dirty {
   296→						ws.w.Flush()
   297→						ws.dirty = false
   298→					}
   299→				}
   300→			}
   301→
   302→			ticker := time.NewTicker(walFlushInterval)
   303→			defer ticker.Stop()
   304→
   305→			// drainBatch processes up to 4096 pending messages without blocking.
   306→			// Returns false if the channel was closed.
   307→			drainBatch := func() bool {
   308→				for range 4096 {
   309→					select {
   310→					case msg, ok := <-msgCh:
   311→						if !ok {
   312→							flushDirty()
   313→							return false
   314→						}
   315→						processMsg(msg)
   316→					case req := <-rotateCh:
   317→						processRotate(req)
   318→					default:
   319→						return true
   320→					}
   321→				}
   322→				return true
   323→			}
   324→
   325→			for {
   326→				select {
   327→				case <-ctx.Done():
   328→					// On shutdown, skip draining buffered messages — a full binary
   329→					// checkpoint will be written from in-memory state, making
   330→					// buffered WAL records redundant.
   331→					flushDirty()
   332→					return
   333→				case msg, ok := <-msgCh:
   334→					if !ok {
   335→						return
   336→					}
   337→					processMsg(msg)
   338→					if !drainBatch() {
   339→						return
   340→					}
   341→					// No flush here — timer handles periodic flushing.
   342→				case <-ticker.C:
   343→					flushDirty()
   344→				case req := <-rotateCh:
   345→					processRotate(req)
   346→				}
   347→			}
   348→		})
   349→	}
   350→}
   351→
   352→// WaitForWALStagingDrain blocks until all WALStaging goroutines have exited.
   353→// Must be called after closing walShardChs to ensure all file handles are
   354→// flushed and closed before checkpoint writes begin.
   355→func WaitForWALStagingDrain() {
   356→	walStagingWg.Wait()
   357→}
   358→
   359→// RotateWALFiles sends rotation requests for the given host directories
   360→// and blocks until all rotations complete. Each request is routed to the
   361→// shard that owns the host directory.
   362→//
   363→// If shutdown is in progress (WAL staging goroutines may have exited),
   364→// rotation is skipped to avoid deadlocking on abandoned channels.
   365→func RotateWALFiles(hostDirs []string) {
   366→	if walShardRotateChs == nil || walShuttingDown.Load() {
   367→		return
   368→	}
   369→	deadline := time.After(2 * time.Minute)
   370→	dones := make([]chan struct{}, 0, len(hostDirs))
   371→	for _, dir := range hostDirs {
   372→		done := make(chan struct{})
   373→		shard := walShardIndexFromDir(dir)
   374→		select {
   375→		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
   376→			dones = append(dones, done)
   377→		case <-deadline:
   378→			cclog.Warnf("[METRICSTORE]> WAL rotation send timed out, %d of %d hosts remaining",
   379→				len(hostDirs)-len(dones), len(hostDirs))
   380→			goto waitDones
   381→		}
   382→	}
   383→waitDones:
   384→	for _, done := range dones {
   385→		select {
   386→		case <-done:
   387→		case <-time.After(30 * time.Second):
   388→			cclog.Warn("[METRICSTORE]> WAL rotation completion timed out, continuing")
   389→			return
   390→		}
   391→	}
   392→}
   393→
   394→// walShardIndexFromDir extracts cluster and node from a host directory path
   395→// and returns the shard index. The path format is: rootDir/cluster/node
   396→func walShardIndexFromDir(hostDir string) int {
   397→	// hostDir = .../cluster/node — extract last two path components
   398→	node := path.Base(hostDir)
   399→	cluster := path.Base(path.Dir(hostDir))
   400→	return walShardIndex(cluster, node)

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


   400→	return walShardIndex(cluster, node)
   401→}
   402→
   403→// RotateWALFilesAfterShutdown directly removes current.wal files for the given
   404→// host directories. Used after shutdown, when WALStaging goroutines have already
   405→// exited and the channel-based RotateWALFiles is no longer safe to call.
   406→func RotateWALFilesAfterShutdown(hostDirs []string) {
   407→	for _, dir := range hostDirs {
   408→		walPath := path.Join(dir, "current.wal")
   409→		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   410→			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   411→		}
   412→	}
   413→}
   414→
   415→// writeWALRecordDirect encodes a WAL record into a contiguous buffer first,
   416→// then writes it to the bufio.Writer in a single call. This prevents partial
   417→// records in the write buffer if a write error occurs mid-record (e.g. disk full).
   418→// Returns the number of bytes written and any error.
   419→func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) (int, error) {
   420→	// Compute payload and total record size.
   421→	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
   422→	for _, s := range msg.Selector {
   423→		payloadSize += 1 + len(s)
   424→	}
   425→	// Total: 8 (header) + payload + 4 (CRC).
   426→	totalSize := 8 + payloadSize + 4
   427→
   428→	// Use stack buffer for typical small records, heap-allocate only for large ones.
   429→	var stackBuf [256]byte
   430→	var buf []byte
   431→	if totalSize <= len(stackBuf) {
   432→		buf = stackBuf[:totalSize]
   433→	} else {
   434→		buf = make([]byte, totalSize)
   435→	}
   436→
   437→	// Header: magic + payload length.
   438→	binary.LittleEndian.PutUint32(buf[0:4], walRecordMagic)
   439→	binary.LittleEndian.PutUint32(buf[4:8], uint32(payloadSize))
   440→
   441→	// Payload starts at offset 8.
   442→	p := 8
   443→
   444→	// Timestamp (8 bytes).
   445→	binary.LittleEndian.PutUint64(buf[p:p+8], uint64(msg.Timestamp))
   446→	p += 8
   447→
   448→	// Metric name length (2 bytes) + metric name.
   449→	binary.LittleEndian.PutUint16(buf[p:p+2], uint16(len(msg.MetricName)))
   450→	p += 2
   451→	p += copy(buf[p:], msg.MetricName)
   452→
   453→	// Selector count (1 byte).
   454→	buf[p] = byte(len(msg.Selector))
   455→	p++
   456→
   457→	// Selectors (1-byte length + bytes each).
   458→	for _, sel := range msg.Selector {
   459→		buf[p] = byte(len(sel))
   460→		p++
   461→		p += copy(buf[p:], sel)
   462→	}
   463→
   464→	// Value (4 bytes, float32 bits).
   465→	binary.LittleEndian.PutUint32(buf[p:p+4], math.Float32bits(float32(msg.Value)))
   466→	p += 4
   467→
   468→	// CRC32 over payload (bytes 8..8+payloadSize).
   469→	crc := crc32.ChecksumIEEE(buf[8 : 8+payloadSize])
   470→	binary.LittleEndian.PutUint32(buf[p:p+4], crc)
   471→
   472→	// Single atomic write to the buffered writer.
   473→	n, err := w.Write(buf)
   474→	return n, err
   475→}
   476→
   477→// readWALRecord reads one WAL record from the reader.
   478→// Returns (nil, nil) on clean EOF. Returns error on data corruption.
   479→// A CRC mismatch indicates a truncated trailing record (expected on crash).
   480→func readWALRecord(r io.Reader) (*WALMessage, error) {
   481→	var magic uint32
   482→	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
   483→		if err == io.EOF {
   484→			return nil, nil // Clean EOF
   485→		}
   486→		return nil, fmt.Errorf("read record magic: %w", err)
   487→	}
   488→
   489→	if magic != walRecordMagic {
   490→		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
   491→	}
   492→
   493→	var payloadLen uint32
   494→	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
   495→		return nil, fmt.Errorf("read payload length: %w", err)
   496→	}
   497→
   498→	if payloadLen > 1<<20 { // 1 MB sanity limit
   499→		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
   500→	}
   501→
   502→	payload := make([]byte, payloadLen)
   503→	if _, err := io.ReadFull(r, payload); err != nil {
   504→		return nil, fmt.Errorf("read payload: %w", err)
   505→	}
   506→
   507→	var storedCRC uint32
   508→	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
   509→		return nil, fmt.Errorf("read CRC: %w", err)
   510→	}
   511→
   512→	if crc32.ChecksumIEEE(payload) != storedCRC {
   513→		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
   514→	}
   515→
   516→	return parseWALPayload(payload)
   517→}
   518→
   519→// parseWALPayload decodes a binary payload into a WALMessage.
   520→func parseWALPayload(payload []byte) (*WALMessage, error) {
   521→	if len(payload) < 8+2+1+4 {
   522→		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
   523→	}
   524→
   525→	offset := 0
   526→
   527→	// Timestamp (8 bytes)
   528→	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
   529→	offset += 8
   530→
   531→	// Metric name (2-byte length + bytes)
   532→	if offset+2 > len(payload) {
   533→		return nil, fmt.Errorf("metric name length overflows payload")
   534→	}
   535→	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
   536→	offset += 2
   537→
   538→	if offset+mLen > len(payload) {
   539→		return nil, fmt.Errorf("metric name overflows payload")
   540→	}
   541→	metricName := string(payload[offset : offset+mLen])
   542→	offset += mLen
   543→
   544→	// Selector count (1 byte)
   545→	if offset >= len(payload) {
   546→		return nil, fmt.Errorf("selector count overflows payload")
   547→	}
   548→	selCount := int(payload[offset])
   549→	offset++
   550→
   551→	selectors := make([]string, selCount)
   552→	for i := range selCount {
   553→		if offset >= len(payload) {
   554→			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
   555→		}
   556→		sLen := int(payload[offset])
   557→		offset++
   558→
   559→		if offset+sLen > len(payload) {
   560→			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
   561→		}
   562→		selectors[i] = string(payload[offset : offset+sLen])
   563→		offset += sLen
   564→	}
   565→
   566→	// Value (4 bytes, float32 bits)
   567→	if offset+4 > len(payload) {
   568→		return nil, fmt.Errorf("value overflows payload")
   569→	}
   570→	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
   571→	value := schema.Float(math.Float32frombits(bits))
   572→
   573→	return &WALMessage{
   574→		MetricName: metricName,
   575→		Timestamp:  ts,
   576→		Selector:   selectors,
   577→		Value:      value,
   578→	}, nil
   579→}
   580→
   581→// loadWALFile reads a WAL file and replays all valid records into the Level tree.
   582→// l is the host-level node. Corrupt or partial trailing records are silently skipped
   583→// (expected on crash). Records older than 'from' are skipped.
   584→func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
   585→	br := bufio.NewReader(f)
   586→
   587→	// Verify file header magic.
   588→	var fileMagic uint32
   589→	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
   590→		if err == io.EOF {
   591→			return nil // Empty file, no data
   592→		}
   593→		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
   594→	}
   595→
   596→	if fileMagic != walFileMagic {
   597→		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
   598→	}
   599→
   600→	// Cache level lookups to avoid repeated tree traversal.
   601→	lvlCache := make(map[string]*Level)
   602→
   603→	for {
   604→		msg, err := readWALRecord(br)
   605→		if err != nil {
   606→			// Truncated trailing record is expected after a crash; stop replaying.
   607→			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
   608→			break
   609→		}
   610→		if msg == nil {
   611→			break // Clean EOF
   612→		}
   613→
   614→		if msg.Timestamp < from {
   615→			continue // Older than retention window
   616→		}
   617→
   618→		minfo, ok := m.Metrics[msg.MetricName]
   619→		if !ok {
   620→			continue // Unknown metric (config may have changed)
   621→		}
   622→
   623→		// Cache key is the null-separated selector path.
   624→		cacheKey := joinSelector(msg.Selector)
   625→		lvl, ok := lvlCache[cacheKey]
   626→		if !ok {
   627→			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
   628→			lvlCache[cacheKey] = lvl
   629→		}
   630→
   631→		// Write directly to the buffer, same as WriteToLevel but without the
   632→		// global level lookup (we already have the right level).
   633→		lvl.lock.Lock()
   634→		b := lvl.metrics[minfo.offset]
   635→		if b == nil {
   636→			b = newBuffer(msg.Timestamp, minfo.Frequency)
   637→			lvl.metrics[minfo.offset] = b
   638→		}
   639→		nb, writeErr := b.write(msg.Timestamp, msg.Value)
   640→		if writeErr == nil && b != nb {
   641→			lvl.metrics[minfo.offset] = nb
   642→		}
   643→		// Ignore write errors for timestamps before buffer start (can happen when
   644→		// replaying WAL entries that predate a loaded snapshot's start time).
   645→		lvl.lock.Unlock()
   646→	}
   647→
   648→	return nil
   649→}
   650→
   651→// joinSelector builds a cache key from a selector slice using null bytes as separators.
   652→func joinSelector(sel []string) string {
   653→	if len(sel) == 0 {
   654→		return ""
   655→	}
   656→	var result strings.Builder
   657→	result.WriteString(sel[0])
   658→	for i := 1; i < len(sel); i++ {
   659→		result.WriteString("\x00" + sel[i])
   660→	}
   661→	return result.String()
   662→}
   663→
   664→// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
   665→// Returns the number of files written, the list of host directories that were
   666→// successfully checkpointed (for WAL rotation), and any errors.
   667→func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
   668→	// Collect all cluster/host pairs.
   669→	m.root.lock.RLock()
   670→	totalHosts := 0
   671→	for _, l1 := range m.root.children {
   672→		l1.lock.RLock()
   673→		totalHosts += len(l1.children)
   674→		l1.lock.RUnlock()
   675→	}
   676→	m.root.lock.RUnlock()
   677→
   678→	levels := make([]*Level, 0, totalHosts)
   679→	selectors := make([][]string, 0, totalHosts)
   680→
   681→	m.root.lock.RLock()
   682→	for sel1, l1 := range m.root.children {
   683→		l1.lock.RLock()
   684→		for sel2, l2 := range l1.children {
   685→			levels = append(levels, l2)
   686→			selectors = append(selectors, []string{sel1, sel2})
   687→		}
   688→		l1.lock.RUnlock()
   689→	}
   690→	m.root.lock.RUnlock()
   691→
   692→	type workItem struct {
   693→		level    *Level
   694→		hostDir  string
   695→		selector []string
   696→	}
   697→
   698→	totalWork := len(levels)
   699→	cclog.Infof("[METRICSTORE]> Starting binary checkpoint for %d hosts with %d workers", totalWork, Keys.NumWorkers)
   700→
   701→	n, errs, completed := int32(0), int32(0), int32(0)
   702→	var successDirs []string
   703→	var successMu sync.Mutex
   704→
   705→	var wg sync.WaitGroup
   706→	wg.Add(Keys.NumWorkers)
   707→	work := make(chan workItem, Keys.NumWorkers*2)
   708→
   709→	// Progress logging goroutine.
   710→	stopProgress := make(chan struct{})
   711→	go func() {
   712→		ticker := time.NewTicker(10 * time.Second)
   713→		defer ticker.Stop()
   714→		for {
   715→			select {
   716→			case <-ticker.C:
   717→				cclog.Infof("[METRICSTORE]> Checkpoint progress: %d/%d hosts (%d written, %d errors)",
   718→					atomic.LoadInt32(&completed), totalWork, atomic.LoadInt32(&n), atomic.LoadInt32(&errs))
   719→			case <-stopProgress:
   720→				return
   721→			}
   722→		}
   723→	}()
   724→
   725→	for range Keys.NumWorkers {
   726→		go func() {
   727→			defer wg.Done()
   728→			for wi := range work {
   729→				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
   730→				if err != nil {
   731→					if err == ErrNoNewArchiveData {
   732→						atomic.AddInt32(&completed, 1)
   733→						continue
   734→					}
   735→					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
   736→					atomic.AddInt32(&errs, 1)
   737→				} else {
   738→					atomic.AddInt32(&n, 1)
   739→					successMu.Lock()
   740→					successDirs = append(successDirs, wi.hostDir)
   741→					successMu.Unlock()
   742→				}
   743→				atomic.AddInt32(&completed, 1)
   744→			}
   745→		}()
   746→	}
   747→
   748→	for i := range levels {
   749→		hostDir := path.Join(dir, path.Join(selectors[i]...))
   750→		work <- workItem{
   751→			level:    levels[i],
   752→			hostDir:  hostDir,
   753→			selector: selectors[i],
   754→		}
   755→	}
   756→	close(work)
   757→	wg.Wait()
   758→	close(stopProgress)
   759→
   760→	if errs > 0 {
   761→		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
   762→	}
   763→	return int(n), successDirs, nil
   764→}
   765→
   766→// toCheckpointBinary writes a binary snapshot file for a single host-level node.
   767→// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
   768→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   769→	cf, err := l.toCheckpointFile(from, to, m)
   770→	if err != nil {
   771→		return err
   772→	}
   773→	if cf == nil {
   774→		return ErrNoNewArchiveData
   775→	}
   776→
   777→	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
   778→		return fmt.Errorf("mkdir %s: %w", dir, err)
   779→	}
   780→
   781→	// Write to a temp file first, then rename (atomic on POSIX).
   782→	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
   783→	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))
   784→
   785→	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   786→	if err != nil {
   787→		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
   788→	}
   789→
   790→	bw := bufio.NewWriter(f)
   791→	if err := writeBinarySnapshotFile(bw, cf); err != nil {
   792→		f.Close()
   793→		os.Remove(tmpPath)
   794→		return fmt.Errorf("write binary snapshot: %w", err)
   795→	}
   796→	if err := bw.Flush(); err != nil {
   797→		f.Close()
   798→		os.Remove(tmpPath)
   799→		return err

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


   800→	}
   801→	f.Close()
   802→
   803→	return os.Rename(tmpPath, finalPath)
   804→}
   805→
   806→// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
   807→func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
   808→	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
   809→		return err
   810→	}
   811→	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
   812→		return err
   813→	}
   814→	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
   815→		return err
   816→	}
   817→	return writeBinaryLevel(w, cf)
   818→}
   819→
   820→// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
   821→func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
   822→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
   823→		return err
   824→	}
   825→
   826→	for name, metric := range cf.Metrics {
   827→		if err := writeString16(w, name); err != nil {
   828→			return err
   829→		}
   830→		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
   831→			return err
   832→		}
   833→		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
   834→			return err
   835→		}
   836→		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
   837→			return err
   838→		}
   839→		for _, v := range metric.Data {
   840→			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
   841→				return err
   842→			}
   843→		}
   844→	}
   845→
   846→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
   847→		return err
   848→	}
   849→
   850→	for name, child := range cf.Children {
   851→		if err := writeString16(w, name); err != nil {
   852→			return err
   853→		}
   854→		if err := writeBinaryLevel(w, child); err != nil {
   855→			return err
   856→		}
   857→	}
   858→
   859→	return nil
   860→}
   861→
   862→// writeString16 writes a 2-byte length-prefixed string to w.
   863→func writeString16(w io.Writer, s string) error {
   864→	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
   865→		return err
   866→	}
   867→	_, err := io.WriteString(w, s)
   868→	return err
   869→}
   870→
   871→// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
   872→// The retention check (from) is applied to the file's 'to' timestamp.
   873→func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
   874→	br := bufio.NewReader(f)
   875→
   876→	var magic uint32
   877→	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
   878→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
   879→	}
   880→	if magic != snapFileMagic {
   881→		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
   882→	}
   883→
   884→	var fileFrom, fileTo int64
   885→	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
   886→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
   887→	}
   888→	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
   889→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
   890→	}
   891→
   892→	if fileTo != 0 && fileTo < from {
   893→		return nil // File is older than retention window, skip it
   894→	}
   895→
   896→	cf, err := readBinaryLevel(br)
   897→	if err != nil {
   898→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
   899→	}
   900→	cf.From = fileFrom
   901→	cf.To = fileTo
   902→
   903→	return l.loadFile(cf, m)
   904→}
   905→
   906→// readBinaryLevel recursively reads a level from the binary snapshot format.
   907→func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
   908→	cf := &CheckpointFile{
   909→		Metrics:  make(map[string]*CheckpointMetrics),
   910→		Children: make(map[string]*CheckpointFile),
   911→	}
   912→
   913→	var numMetrics uint32
   914→	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
   915→		return nil, fmt.Errorf("read num_metrics: %w", err)
   916→	}
   917→
   918→	for range numMetrics {
   919→		name, err := readString16(r)
   920→		if err != nil {
   921→			return nil, fmt.Errorf("read metric name: %w", err)
   922→		}
   923→
   924→		var freq, start int64
   925→		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
   926→			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
   927→		}
   928→		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
   929→			return nil, fmt.Errorf("read start for %s: %w", name, err)
   930→		}
   931→
   932→		var numValues uint32
   933→		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
   934→			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
   935→		}
   936→
   937→		data := make([]schema.Float, numValues)
   938→		for i := range numValues {
   939→			var bits uint32
   940→			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
   941→				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
   942→			}
   943→			data[i] = schema.Float(math.Float32frombits(bits))
   944→		}
   945→
   946→		cf.Metrics[name] = &CheckpointMetrics{
   947→			Frequency: freq,
   948→			Start:     start,
   949→			Data:      data,
   950→		}
   951→	}
   952→
   953→	var numChildren uint32
   954→	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
   955→		return nil, fmt.Errorf("read num_children: %w", err)
   956→	}
   957→
   958→	for range numChildren {
   959→		childName, err := readString16(r)
   960→		if err != nil {
   961→			return nil, fmt.Errorf("read child name: %w", err)
   962→		}
   963→
   964→		child, err := readBinaryLevel(r)
   965→		if err != nil {
   966→			return nil, fmt.Errorf("read child %s: %w", childName, err)
   967→		}
   968→		cf.Children[childName] = child
   969→	}
   970→
   971→	return cf, nil
   972→}
   973→
   974→// readString16 reads a 2-byte length-prefixed string from r.
   975→func readString16(r io.Reader) (string, error) {
   976→	var sLen uint16
   977→	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
   978→		return "", err
   979→	}
   980→	buf := make([]byte, sLen)
   981→	if _, err := io.ReadFull(r, buf); err != nil {
   982→		return "", err
   983→	}
   984→	return string(buf), nil
   985→}
   986→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file contains the API types and data fetching logic for querying metric data
     7→// from the in-memory metric store. It provides structures for building complex queries
     8→// with support for aggregation, scaling, padding, and statistics computation.
     9→package metricstore
    10→
    11→import (
    12→	"errors"
    13→	"fmt"
    14→	"math"
    15→
    16→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    17→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    18→	"github.com/ClusterCockpit/cc-lib/v2/util"
    19→)
    20→
    21→var (
    22→	// ErrNoHostOrMetric is returned when the metric store does not find the host or the metric
    23→	ErrNoHostOrMetric error = errors.New("[METRICSTORE]> metric or host not found")
    24→	// ErrInvalidTimeRange is returned when a query has 'from' >= 'to'
    25→	ErrInvalidTimeRange = errors.New("[METRICSTORE]> invalid time range: 'from' must be before 'to'")
    26→	// ErrEmptyCluster is returned when a query with ForAllNodes has no cluster specified
    27→	ErrEmptyCluster = errors.New("[METRICSTORE]> cluster name cannot be empty")
    28→)
    29→
    30→// APIMetricData represents the response data for a single metric query.
    31→//
    32→// It contains both the time-series data points and computed statistics (avg, min, max).
    33→// If an error occurred during data retrieval, the Error field will be set and other
    34→// fields may be incomplete.
    35→type APIMetricData struct {
    36→	Error      *string           `json:"error,omitempty"`
    37→	Data       schema.FloatArray `json:"data,omitempty"`
    38→	From       int64             `json:"from"`
    39→	To         int64             `json:"to"`
    40→	Resolution int64             `json:"resolution"`
    41→	Avg        schema.Float      `json:"avg"`
    42→	Min        schema.Float      `json:"min"`
    43→	Max        schema.Float      `json:"max"`
    44→}
    45→
    46→// APIQueryRequest represents a batch query request for metric data.
    47→//
    48→// It supports two modes of operation:
    49→//  1. Explicit queries via the Queries field
    50→//  2. Automatic query generation via ForAllNodes (queries all specified metrics for all nodes in the cluster)
    51→//
    52→// The request can be customized with flags to include/exclude statistics, raw data, and padding.
    53→type APIQueryRequest struct {
    54→	Cluster     string     `json:"cluster"`
    55→	Queries     []APIQuery `json:"queries"`
    56→	ForAllNodes []string   `json:"for-all-nodes"`
    57→	From        int64      `json:"from"`
    58→	To          int64      `json:"to"`
    59→	WithStats   bool       `json:"with-stats"`
    60→	WithData    bool       `json:"with-data"`
    61→	WithPadding bool       `json:"with-padding"`
    62→}
    63→
    64→// APIQueryResponse represents the response to an APIQueryRequest.
    65→//
    66→// Results is a 2D array where each outer element corresponds to a query,
    67→// and each inner element corresponds to a selector within that query
    68→// (e.g., multiple CPUs or cores).
    69→type APIQueryResponse struct {
    70→	Queries []APIQuery        `json:"queries,omitempty"`
    71→	Results [][]APIMetricData `json:"results"`
    72→}
    73→
    74→// APIQuery represents a single metric query with optional hierarchical selectors.
    75→//
    76→// The hierarchical selection works as follows:
    77→//   - Hostname: The node to query
    78→//   - Type + TypeIds: First level of hierarchy (e.g., "cpu" + ["0", "1", "2"])
    79→//   - SubType + SubTypeIds: Second level of hierarchy (e.g., "core" + ["0", "1"])
    80→//
    81→// If Aggregate is true, data from multiple type/subtype IDs will be aggregated according
    82→// to the metric's aggregation strategy. Otherwise, separate results are returned for each combination.
    83→type APIQuery struct {
    84→	Type        *string      `json:"type,omitempty"`
    85→	SubType     *string      `json:"subtype,omitempty"`
    86→	Metric      string       `json:"metric"`
    87→	Hostname    string       `json:"host"`
    88→	Resolution  int64        `json:"resolution"`
    89→	TypeIds     []string     `json:"type-ids,omitempty"`
    90→	SubTypeIds  []string     `json:"subtype-ids,omitempty"`
    91→	ScaleFactor schema.Float `json:"scale-by,omitempty"`
    92→	Aggregate   bool         `json:"aggreg"`
    93→}
    94→
    95→// AddStats computes and populates the Avg, Min, and Max fields from the Data array.
    96→//
    97→// NaN values in the data are ignored during computation. If all values are NaN,
    98→// the statistics fields will be set to NaN.
    99→//
   100→// TODO: Optimize this, just like the stats endpoint!
   101→func (data *APIMetricData) AddStats() {
   102→	n := 0
   103→	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
   104→	for _, x := range data.Data {
   105→		if x.IsNaN() {
   106→			continue
   107→		}
   108→
   109→		n += 1
   110→		sum += float64(x)
   111→		min = math.Min(min, float64(x))
   112→		max = math.Max(max, float64(x))
   113→	}
   114→
   115→	if n > 0 {
   116→		avg := sum / float64(n)
   117→		data.Avg = schema.Float(avg)
   118→		data.Min = schema.Float(min)
   119→		data.Max = schema.Float(max)
   120→	} else {
   121→		data.Avg, data.Min, data.Max = schema.NaN, schema.NaN, schema.NaN
   122→	}
   123→}
   124→
   125→// ScaleBy multiplies all data points and statistics by the given factor.
   126→//
   127→// This is commonly used for unit conversion (e.g., bytes to gigabytes).
   128→// Scaling by 0 or 1 is a no-op for performance reasons.
   129→func (data *APIMetricData) ScaleBy(f schema.Float) {
   130→	if f == 1 {
   131→		return
   132→	}
   133→
   134→	data.Avg *= f
   135→	data.Min *= f
   136→	data.Max *= f
   137→	for i := 0; i < len(data.Data); i++ {
   138→		data.Data[i] *= f
   139→	}
   140→}
   141→
   142→// PadDataWithNull pads the beginning of the data array with NaN values if needed.
   143→//
   144→// This ensures that the data aligns with the requested 'from' timestamp, even if
   145→// the metric store doesn't have data for the earliest time points. This is useful
   146→// for maintaining consistent array indexing across multiple queries.
   147→//
   148→// Parameters:
   149→//   - ms: MemoryStore instance to lookup metric configuration
   150→//   - from: The requested start timestamp
   151→//   - to: The requested end timestamp (unused but kept for API consistency)
   152→//   - metric: The metric name to lookup frequency information
   153→func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metric string) {
   154→	minfo, ok := ms.Metrics[metric]
   155→	if !ok {
   156→		return
   157→	}
   158→
   159→	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
   160→		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
   161→		ndata := make([]schema.Float, 0, padfront+len(data.Data))
   162→		for range padfront {
   163→			ndata = append(ndata, schema.NaN)
   164→		}
   165→		for j := 0; j < len(data.Data); j++ {
   166→			ndata = append(ndata, data.Data[j])
   167→		}
   168→		data.Data = ndata
   169→	}
   170→}
   171→
   172→// FetchData executes a batch metric query request and returns the results.
   173→//
   174→// This is the primary API for retrieving metric data from the memory store. It supports:
   175→//   - Individual queries via req.Queries
   176→//   - Batch queries for all nodes via req.ForAllNodes
   177→//   - Hierarchical selector construction (cluster → host → type → subtype)
   178→//   - Optional statistics computation (avg, min, max)
   179→//   - Optional data scaling
   180→//   - Optional data padding with NaN values
   181→//
   182→// The function constructs selectors based on the query parameters and calls MemoryStore.Read()
   183→// for each selector. If a query specifies Aggregate=false with multiple type/subtype IDs,
   184→// separate results are returned for each combination.
   185→//
   186→// Parameters:
   187→//   - req: The query request containing queries, time range, and options
   188→//
   189→// Returns:
   190→//   - APIQueryResponse containing results for each query, or error if validation fails
   191→//
   192→// Errors:
   193→//   - ErrInvalidTimeRange if req.From > req.To
   194→//   - ErrEmptyCluster if req.ForAllNodes is used without specifying a cluster
   195→//   - Error if MemoryStore is not initialized
   196→//   - Individual query errors are stored in APIMetricData.Error field
   197→func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
   198→	if req.From > req.To {
   199→		return nil, ErrInvalidTimeRange
   200→	}

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements ingestion of InfluxDB line-protocol metric data received
     7→// over NATS. Each line encodes one metric sample with the following structure:
     8→//
     9→//	<measurement>[,cluster=<c>][,hostname=<h>][,type=<t>][,type-id=<id>][,subtype=<s>][,stype-id=<id>] value=<v> [<timestamp>]
    10→//
    11→// The measurement name identifies the metric (e.g. "cpu_load"). Tags provide
    12→// routing information (cluster, host) and optional sub-device selectors (type,
    13→// subtype). Only one field is expected per line: "value".
    14→//
    15→// After decoding, each sample is:
    16→//  1. Written to the in-memory store via ms.WriteToLevel.
    17→//  2. If the checkpoint format is "wal", also forwarded to the WAL staging
    18→//     goroutine via the WALMessages channel for durable write-ahead logging.
    19→package metricstore
    20→
    21→import (
    22→	"bytes"
    23→	"context"
    24→	"fmt"
    25→	"sync"
    26→	"sync/atomic"
    27→	"time"
    28→
    29→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    30→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    31→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    32→	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
    33→)
    34→
    35→// ReceiveNats subscribes to all configured NATS subjects and feeds incoming
    36→// line-protocol messages into the MemoryStore.
    37→//
    38→// When workers > 1 a pool of goroutines drains a shared channel so that
    39→// multiple messages can be decoded in parallel. With workers == 1 the NATS
    40→// callback decodes inline (no channel overhead, lower latency).
    41→//
    42→// The function blocks until ctx is cancelled and all worker goroutines have
    43→// finished. It returns nil when the NATS client is not configured; callers
    44→// should treat that as a no-op rather than an error.
    45→func ReceiveNats(ms *MemoryStore,
    46→	workers int,
    47→	ctx context.Context,
    48→) error {
    49→	nc := nats.GetClient()
    50→
    51→	if nc == nil {
    52→		cclog.Warn("NATS client not initialized")
    53→		return nil
    54→	}
    55→
    56→	var wg sync.WaitGroup
    57→	msgs := make(chan []byte, max(workers*256, 8192))
    58→
    59→	for _, sc := range *Keys.Subscriptions {
    60→		clusterTag := sc.ClusterTag
    61→		if workers > 1 {
    62→			wg.Add(workers)
    63→
    64→			for range workers {
    65→				go func() {
    66→					defer wg.Done()
    67→					for m := range msgs {
    68→						dec := lineprotocol.NewDecoderWithBytes(m)
    69→						if err := DecodeLine(dec, ms, clusterTag); err != nil {
    70→							cclog.Errorf("error: %s", err.Error())
    71→						}
    72→					}
    73→				}()
    74→			}
    75→
    76→			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
    77→				select {
    78→				case msgs <- data:
    79→				case <-ctx.Done():
    80→				}
    81→			})
    82→		} else {
    83→			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
    84→				dec := lineprotocol.NewDecoderWithBytes(data)
    85→				if err := DecodeLine(dec, ms, clusterTag); err != nil {
    86→					cclog.Errorf("error: %s", err.Error())
    87→				}
    88→			})
    89→		}
    90→		cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo)
    91→	}
    92→
    93→	go func() {
    94→		<-ctx.Done()
    95→		close(msgs)
    96→	}()
    97→
    98→	wg.Wait()
    99→
   100→	return nil
   101→}
   102→
   103→// reorder prepends prefix to buf in-place when buf has enough spare capacity,
   104→// avoiding an allocation. Falls back to a regular append otherwise.
   105→//
   106→// It is used to assemble the "type<type-id>" and "subtype<stype-id>" selector
   107→// strings when the type tag arrives before the type-id tag in the line, so the
   108→// two byte slices need to be concatenated in tag-declaration order regardless
   109→// of wire order.
   110→func reorder(buf, prefix []byte) []byte {
   111→	n := len(prefix)
   112→	m := len(buf)
   113→	if cap(buf) < m+n {
   114→		return append(prefix[:n:n], buf...)
   115→	} else {
   116→		buf = buf[:n+m]
   117→		for i := m - 1; i >= 0; i-- {
   118→			buf[i+n] = buf[i]
   119→		}
   120→		for i := range n {
   121→			buf[i] = prefix[i]
   122→		}
   123→		return buf
   124→	}
   125→}
   126→
   127→// walDropped counts WAL messages dropped due to a full channel.
   128→// Logged periodically; not critical since binary snapshots capture the data.
   129→var walDropped atomic.Int64
   130→
   131→// decodeState holds the per-call scratch buffers used by DecodeLine.
   132→// Instances are recycled via decodeStatePool to avoid repeated allocations
   133→// during high-throughput ingestion.
   134→type decodeState struct {
   135→	// metricBuf holds a copy of the current measurement name (line-protocol
   136→	// measurement field). Copied because dec.Measurement() returns a slice
   137→	// that is invalidated by the next decoder call.
   138→	metricBuf []byte
   139→
   140→	// selector is the sub-device path passed to WriteToLevel and WALMessage
   141→	// (e.g. ["socket0"] or ["socket0", "memctrl1"]). Reused across lines.
   142→	selector []string
   143→
   144→	// typeBuf accumulates the concatenated "type"+"type-id" tag value for the
   145→	// current line. Reset at the start of each line's tag-decode loop.
   146→	typeBuf []byte
   147→
   148→	// subTypeBuf accumulates the concatenated "subtype"+"stype-id" tag value.
   149→	// Reset at the start of each line's tag-decode loop.
   150→	subTypeBuf []byte
   151→
   152→	// prevTypeBytes / prevTypeStr cache the last seen typeBuf content and its
   153→	// string conversion. Because consecutive lines in a batch typically address
   154→	// the same sub-device, the cache hit rate is very high and avoids
   155→	// repeated []byte→string allocations.
   156→	prevTypeBytes []byte
   157→	prevTypeStr   string
   158→
   159→	// prevSubTypeBytes / prevSubTypeStr are the same cache for the subtype.
   160→	prevSubTypeBytes []byte
   161→	prevSubTypeStr   string
   162→}
   163→
   164→// decodeStatePool recycles decodeState values across DecodeLine calls to
   165→// reduce GC pressure during sustained metric ingestion.
   166→var decodeStatePool = sync.Pool{
   167→	New: func() any {
   168→		return &decodeState{
   169→			metricBuf:  make([]byte, 0, 16),
   170→			selector:   make([]string, 0, 4),
   171→			typeBuf:    make([]byte, 0, 16),
   172→			subTypeBuf: make([]byte, 0, 16),
   173→		}
   174→	},
   175→}
   176→
   177→// DecodeLine reads all lines from dec (InfluxDB line-protocol) and writes each
   178→// decoded metric sample into ms.
   179→//
   180→// clusterDefault is used as the cluster name for lines that do not carry a
   181→// "cluster" tag. Callers typically supply the ClusterTag value from the NATS
   182→// subscription configuration.
   183→//
   184→// Performance notes:
   185→//   - A decodeState is obtained from decodeStatePool to reuse scratch buffers.
   186→//   - The Level pointer (host-level node in the metric tree) is cached across
   187→//     consecutive lines that share the same cluster+host pair to avoid
   188→//     repeated lock acquisitions on the root and cluster levels.
   189→//   - []byte→string conversions for type/subtype selectors are cached via
   190→//     prevType*/prevSubType* fields because batches typically repeat the same
   191→//     sub-device identifiers.
   192→//   - Timestamp parsing tries Second precision first; if that fails it retries
   193→//     Millisecond, Microsecond, and Nanosecond in turn. A missing timestamp
   194→//     falls back to time.Now().
   195→//
   196→// When the checkpoint format is "wal" each successfully decoded sample is also
   197→// sent to WALMessages so the WAL staging goroutine can persist it durably
   198→// before the next binary snapshot.
   199→func DecodeLine(dec *lineprotocol.Decoder,
   200→	ms *MemoryStore,

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"bufio"
    10→	"encoding/binary"
    11→	"encoding/json"
    12→	"fmt"
    13→	"os"
    14→	"path/filepath"
    15→	"sort"
    16→
    17→	pq "github.com/parquet-go/parquet-go"
    18→)
    19→
    20→// ParquetMetricRow is the long-format schema for archived metric data.
    21→// One row per (host, metric, scope, scope_id, timestamp) data point.
    22→// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
    23→type ParquetMetricRow struct {
    24→	Cluster   string  `parquet:"cluster"`
    25→	Hostname  string  `parquet:"hostname"`
    26→	Metric    string  `parquet:"metric"`
    27→	Scope     string  `parquet:"scope"`
    28→	ScopeID   string  `parquet:"scope_id"`
    29→	Timestamp int64   `parquet:"timestamp"`
    30→	Frequency int64   `parquet:"frequency"`
    31→	Value     float32 `parquet:"value"`
    32→}
    33→
    34→// parseScopeFromName infers scope and scope_id from a child level name.
    35→// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
    36→// "a0" (accelerator) → ("accelerator", "0").
    37→// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
    38→func parseScopeFromName(name string) (string, string) {
    39→	prefixes := []struct {
    40→		prefix string
    41→		scope  string
    42→	}{
    43→		{"socket", "socket"},
    44→		{"memoryDomain", "memoryDomain"},
    45→		{"core", "core"},
    46→		{"hwthread", "hwthread"},
    47→		{"cpu", "hwthread"},
    48→		{"accelerator", "accelerator"},
    49→	}
    50→
    51→	for _, p := range prefixes {
    52→		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
    53→			id := name[len(p.prefix):]
    54→			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
    55→				return p.scope, id
    56→			}
    57→		}
    58→	}
    59→
    60→	return name, ""
    61→}
    62→
    63→// parquetArchiveWriter supports incremental writes to a Parquet file.
    64→// Uses streaming writes to avoid accumulating all rows in memory.
    65→type parquetArchiveWriter struct {
    66→	writer *pq.GenericWriter[ParquetMetricRow]
    67→	bw     *bufio.Writer
    68→	f      *os.File
    69→	batch  []ParquetMetricRow // reusable batch buffer
    70→	count  int
    71→}
    72→
    73→const parquetBatchSize = 1024
    74→
    75→// newParquetArchiveWriter creates a streaming Parquet writer with Zstd compression.
    76→func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
    77→	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
    78→		return nil, fmt.Errorf("creating archive directory: %w", err)
    79→	}
    80→
    81→	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
    82→	if err != nil {
    83→		return nil, fmt.Errorf("creating parquet file: %w", err)
    84→	}
    85→
    86→	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer
    87→
    88→	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
    89→		pq.Compression(&pq.Zstd),
    90→	)
    91→
    92→	return &parquetArchiveWriter{
    93→		writer: writer,
    94→		bw:     bw,
    95→		f:      f,
    96→		batch:  make([]ParquetMetricRow, 0, parquetBatchSize),
    97→	}, nil
    98→}
    99→
   100→// WriteCheckpointFile streams a CheckpointFile tree directly to Parquet rows,
   101→// writing metrics in sorted order without materializing all rows in memory.
   102→// Call FlushRowGroup() after writing all checkpoint files for a host.
   103→func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {
   104→	w.writeLevel(cf, cluster, hostname, scope, scopeID)
   105→
   106→	// Flush remaining batch
   107→	if len(w.batch) > 0 {
   108→		if _, err := w.writer.Write(w.batch); err != nil {
   109→			return fmt.Errorf("writing parquet rows: %w", err)
   110→		}
   111→		w.count += len(w.batch)
   112→		w.batch = w.batch[:0]
   113→	}
   114→
   115→	return nil
   116→}
   117→
   118→// FlushRowGroup flushes the current row group to the Parquet file.
   119→// Should be called once per host after all checkpoint files for that host are written.
   120→func (w *parquetArchiveWriter) FlushRowGroup() error {
   121→	if err := w.writer.Flush(); err != nil {
   122→		return fmt.Errorf("flushing parquet row group: %w", err)
   123→	}
   124→	return nil
   125→}
   126→
   127→// writeLevel recursively writes metrics from a CheckpointFile level.
   128→// Metric names and child names are sorted for deterministic, compression-friendly output.
   129→func (w *parquetArchiveWriter) writeLevel(cf *CheckpointFile, cluster, hostname, scope, scopeID string) {
   130→	// Sort metric names for deterministic order
   131→	metricNames := make([]string, 0, len(cf.Metrics))
   132→	for name := range cf.Metrics {
   133→		metricNames = append(metricNames, name)
   134→	}
   135→	sort.Strings(metricNames)
   136→
   137→	for _, metricName := range metricNames {
   138→		cm := cf.Metrics[metricName]
   139→		ts := cm.Start
   140→		for _, v := range cm.Data {
   141→			if !v.IsNaN() {
   142→				w.batch = append(w.batch, ParquetMetricRow{
   143→					Cluster:   cluster,
   144→					Hostname:  hostname,
   145→					Metric:    metricName,
   146→					Scope:     scope,
   147→					ScopeID:   scopeID,
   148→					Timestamp: ts,
   149→					Frequency: cm.Frequency,
   150→					Value:     float32(v),

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


Perfect! I now have a comprehensive understanding of the metricstore package. Let me provide you with a complete analysis.

## Comprehensive Metricstore Package Analysis

### 1. Init() Function Signature and Parameters

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go` (lines 125-194)

```go
func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup)
```

**Parameters:**
- `rawConfig`: JSON configuration for the metric store (MetricStoreConfig); may be nil to use defaults
- `metrics`: Map of metric names to their configurations (frequency and aggregation strategy)
- `wg`: WaitGroup that will be incremented for each background goroutine started

**Key Initialization Steps:**
1. Validates and decodes JSON metric store configuration
2. Sets NumWorkers (defaults to min(NumCPU/2+1, 10), max 10)
3. Calls InitMetrics() to create singleton MemoryStore
4. Loads checkpoints within retention window from disk
5. Starts 5 main background workers:
   - `Retention()` - frees old data per retention policy
   - `Checkpointing()` - periodic checkpoint creation
   - `CleanUp()` - archiving or deletion of old checkpoints
   - `WALStaging()` - WAL message processing (if WAL format enabled)
   - `MemoryUsageTracker()` - monitors and enforces memory cap
6. Optional: Subscribes to NATS for real-time metric ingestion

### 2. Shutdown() Function

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go` (lines 274-321)

```go
func Shutdown()
```

**Behavior:**
1. Cancels context to stop all background workers
2. If WAL format: sets shutdownFlag, sleeps 100ms, closes WAL channels, waits for drain
3. Writes final checkpoint (JSON or WAL format)
4. Logs timing information

### 3. Configuration Structs

**MetricStoreConfig** (main config):
```go
type MetricStoreConfig struct {
    NumWorkers        int           // Workers for checkpoint/archive (default: auto)
    RetentionInMemory string        // Duration string (e.g., "48h")
    CheckpointInterval string       // Duration string (default: "12h")
    MemoryCap        int           // Memory cap in GB (triggers forceFree)
    Checkpoints      Checkpoints   // Checkpoint persistence config
    Debug            *Debug        // Dev/profiling options
    Cleanup          *Cleanup      // Long-term storage config
    Subscriptions    *Subscriptions // NATS topics
}
```

**Checkpoints** (persistence):
```go
type Checkpoints struct {
    FileFormat string // "json" or "wal" (default: "wal")
    RootDir    string // Checkpoint directory path
    MaxWALSize int64  // Max WAL file size (0 = unlimited)
}
```

**Cleanup** (archiving):
```go
type Cleanup struct {
    RootDir string // Archive path
    Mode    string // "delete" or "archive"
}
```

**Debug**:
```go
type Debug struct {
    DumpToFile string // Dump checkpoint path
    EnableGops bool   // Enable gops agent
}
```

**Subscriptions**:
```go
type Subscriptions []struct {
    SubscribeTo string // NATS subject name
    ClusterTag  string // Optional default cluster
}
```

**MetricConfig**:
```go
type MetricConfig struct {
    Frequency   int64 // Interval in seconds
    Aggregation AggregationStrategy // sum/avg/none
    offset      int   // Internal index (assigned during Init)
}
```

**AggregationStrategy** (enum):
- `NoAggregation` (0) - Do not aggregate
- `SumAggregation` (1) - Sum values (e.g., power)
- `AvgAggregation` (2) - Average values (e.g., temperature)

### 4. Recent Features/Settings Added

**WAL (Write-Ahead Log) Implementation** (walCheckpoint.go):
- Binary snapshot format (`.bin` files) + append-only WAL (`.wal` files)
- Replaces JSON snapshot with faster, more compact binary format
- Per-host WAL files at `checkpoints/cluster/host/current.wal`
- Magic numbers: `0xCC1DA701` (WAL file), `0xCC1DA7A1` (WAL record), `0xCC5B0001` (binary snapshot)
- Supports WAL rotation and max WAL size limits
- Crash-safe with CRC32 checksums on WAL records

**Memory Usage Tracking** (MemoryUsageTracker):
- Monitors actual heap memory usage (runtime.MemStats)
- Compares against MemoryCap setting
- Progressive emergency buffer freeing at 75%, 50%, 25% retention thresholds
- Adaptive ticker intervals (5 min normal, 30 sec when >80% of cap)

**Selective Buffer Retention** (NodeProvider interface):
- `SetNodeProvider()` allows caller to provide job state information
- `Free()` function consults NodeProvider to preserve data for nodes with active jobs
- `GetUsedNodes()` returns map of cluster→hostnames in active use
- Enables selective retention beyond retention window for running jobs

**Parquet Archiving** (parquetArchive.go):
- Long-format archive: one row per (host, metric, scope, timestamp) point
- Streaming Parquet writer with Zstd compression
- Parquet schema: cluster, hostname, metric, scope, scope_id, timestamp, frequency, value

### 5. MemoryStore Type and Methods

**MemoryStore** struct:
```go
type MemoryStore struct {
    Metrics      map[string]MetricConfig
    root         Level
    nodeProvider NodeProvider
}
```

**Key Public Methods:**
- `Write(selector []string, ts int64, metrics []Metric)` - Write metrics at specific level
- `Read(selector util.Selector, metric string, from, to, resolution int64)` - Query metrics with aggregation
- `GetLevel(selector []string)` - Get/create level at selector path
- `ListChildren(selector []string)` - List child names at selector
- `Free(selector []string, t int64)` - Free buffers older than t
- `ForceFree()` - Unconditionally remove oldest buffers
- `FreeAll()` - Clear all data
- `GetPaths(targetDepth int)` - Get selector paths at specific depth
- `SizeInBytes()` / `SizeInGB()` - Memory usage statistics
- `FromCheckpointFiles(dir string, from int64)` - Load checkpoints at startup
- `ToCheckpoint(dir, from, to int64)` - Write JSON checkpoints
- `ToCheckpointWAL(dir, from, to int64)` - Write binary snapshots + rotate WAL
- `SetNodeProvider(provider NodeProvider)` - Set job state provider

### 6. Checkpoint/Retention/Cleanup/NATS Structures and Functions

**Checkpoint Functions:**
- `Checkpointing(wg, ctx)` - Background worker, runs every 12h (default)
- `FromCheckpoint(dir, from int64)` - Parallel checkpoint loading
- `FromCheckpointFiles(dir, from int64)` - Entry point at startup
- `ToCheckpoint(dir, from, to int64)` - JSON format (parallel with worker pool)
- `ToCheckpointWAL(dir, from, to int64)` - Binary format with WAL rotation

**WAL Functions:**
- `WALStaging(wg, ctx)` - N sharded goroutines, hash-routed per cluster+node
- `SendWALMessage(msg *WALMessage)` - Non-blocking send to WAL channel
- `RotateWALFiles(hostDirs []string)` - Request WAL rotation after snapshot
- `writeWALRecordDirect(w, msg)` - Write single WAL record with CRC
- `readWALRecord(r)` - Read and validate one WAL record
- `loadWALFile(m, f, from int64)` - Replay WAL on startup

**Retention Functions:**
- `Retention(wg, ctx)` - Background worker, runs every RetentionInMemory/2
- `Free(ms *MemoryStore, t time.Time)` - Free buffers, respecting NodeProvider
- `FreeSelected(ms, selectors, t)` - Free specific selector paths only
- `GetSelectors(ms, excludeMap)` - Build list of selectors to free

**Cleanup/Archive Functions:**
- `CleanUp(wg, ctx)` - Background worker, interval = RetentionInMemory
- `CleanupCheckpoints(dir, cleanupDir, from, deleteInstead)` - Delete or archive
- `deleteCheckpoints(dir, from)` - Parallel deletion of old checkpoints
- `archiveCheckpoints(dir, cleanupDir, from)` - Convert to Parquet + delete originals

**Memory Monitoring:**
- `MemoryUsageTracker(wg, ctx)` - Monitors heap usage, force-frees when over cap
- `bufferPool` - PersistentBufferPool for buffer reuse

**NATS Integration:**
- `ReceiveNats(ms, workers, ctx)` - Subscribe to NATS topics, decode line protocol
- `DecodeLine(dec, ms, clusterDefault)` - Parse InfluxDB line protocol
- `Keys.Subscriptions` - Array of NATS subscription configs

**Level (Hierarchical Tree Node):**
```go
type Level struct {
    children map[string]*Level
    metrics  []*buffer  // One per metric
    lock     sync.RWMutex
}
```

Methods: `findLevelOrCreate()`, `findBuffers()`, `findLevel()`, `free()`, `forceFree()`, `sizeInBytes()`, `collectPaths()`

**Buffer (Time-Series Storage):**
```go
type buffer struct {
    prev      *buffer
    next      *buffer
    data      []schema.Float
    frequency int64
    start     int64
    archived  bool
    closed    bool
    lastUsed  int64
}
```

Methods: `write()`, `read()`, `free()`, `forceFreeOldest()`, `iterFromTo()`, linked-list chain with default capacity 512 floats.

---

**File Paths for Reference:**
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go` - Core Init/Shutdown
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go` - Configuration structs
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go` - JSON checkpoint implementation
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go` - WAL + binary snapshot
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go` - Archive/cleanup worker
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/level.go` - Hierarchical tree structure
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/buffer.go` - Time-series buffer chains

agentId: a45167c5a47159b8d (use SendMessage with to: 'a45167c5a47159b8d' to continue this agent)
<usage>total_tokens: 83558
tool_uses: 15
duration_ms: 47169</usage>

Prompt:
Explore the metricstore package in /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/. Read ALL files in that directory. I need to understand:

1. The Init() function signature and all parameters it accepts
2. The Shutdown() function
3. The configuration struct(s) used - what fields they have
4. Any new features or settings added recently
5. The MemoryStore type and its methods
6. Any checkpoint, retention, cleanup, or NATS subscription related structs/functions

Return the full content of all files in that directory.


Now let me read the key files directly to understand exactly what needs to change.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-metric-store.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package main
     7→
     8→import (
     9→	"context"
    10→	"flag"
    11→	"fmt"
    12→	"os"
    13→	"os/signal"
    14→	"runtime/debug"
    15→	"sync"
    16→	"syscall"
    17→
    18→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    19→	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
    20→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    21→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    22→	"github.com/ClusterCockpit/cc-lib/v2/runtime"
    23→	"github.com/ClusterCockpit/cc-metric-store/internal/api"
    24→	"github.com/ClusterCockpit/cc-metric-store/internal/config"
    25→	"github.com/google/gops/agent"
    26→)
    27→
    28→const (
    29→	envGOGC = "GOGC"
    30→)
    31→
    32→var (
    33→	date    string
    34→	commit  string
    35→	version string
    36→)
    37→
    38→var (
    39→	flagGops, flagVersion, flagDev, flagLogDateTime bool
    40→	flagConfigFile, flagLogLevel                    string
    41→)
    42→
    43→func printVersion() {
    44→	fmt.Printf("Version:\t%s\n", version)
    45→	fmt.Printf("Git hash:\t%s\n", commit)
    46→	fmt.Printf("Build time:\t%s\n", date)
    47→}
    48→
    49→func runServer(ctx context.Context) error {
    50→	var wg sync.WaitGroup
    51→
    52→	mscfg := ccconf.GetPackageConfig("metrics")
    53→	if mscfg == nil {
    54→		return fmt.Errorf("missing metrics configuration")
    55→	}
    56→	config.InitMetrics(mscfg)
    57→
    58→	mscfg = ccconf.GetPackageConfig("metric-store")
    59→	if mscfg == nil {
    60→		return fmt.Errorf("missing metricstore configuration")
    61→	}
    62→
    63→	metricstore.Init(mscfg, config.GetMetrics(), &wg)
    64→
    65→	// Set GC percent if not configured
    66→	if os.Getenv(envGOGC) == "" {
    67→		debug.SetGCPercent(15)
    68→	}
    69→
    70→	if config.Keys.BackendURL != "" {
    71→		ms := metricstore.GetMemoryStore()
    72→		ms.SetNodeProvider(api.NewBackendNodeProvider(config.Keys.BackendURL))
    73→		cclog.Infof("Node provider configured with backend URL: %s", config.Keys.BackendURL)
    74→	}
    75→
    76→	// Initialize HTTP server
    77→	srv, err := NewServer(version, commit, date)
    78→	if err != nil {
    79→		return fmt.Errorf("creating server: %w", err)
    80→	}
    81→
    82→	// Channel to collect errors from server
    83→	errChan := make(chan error, 1)
    84→
    85→	// Start HTTP server
    86→	wg.Go(func() {
    87→		if err := srv.Start(ctx); err != nil {
    88→			errChan <- err
    89→		}
    90→	})
    91→
    92→	// Handle shutdown signals
    93→	wg.Add(1)
    94→	sigs := make(chan os.Signal, 1)
    95→	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
    96→	go func() {
    97→		defer wg.Done()
    98→		select {
    99→		case <-sigs:
   100→			cclog.Info("Shutdown signal received")
   101→		case <-ctx.Done():
   102→		}
   103→
   104→		runtime.SystemdNotify(false, "Shutting down ...")
   105→		srv.Shutdown(ctx)
   106→	}()
   107→
   108→	runtime.SystemdNotify(true, "running")
   109→
   110→	// Wait for completion or errors
   111→	go func() {
   112→		wg.Wait()
   113→		close(errChan)
   114→	}()
   115→
   116→	// Wait for either server startup error or shutdown completion
   117→	if err := <-errChan; err != nil {
   118→		return err
   119→	}
   120→
   121→	cclog.Print("Graceful shutdown completed!")
   122→	return nil
   123→}
   124→
   125→func run() error {
   126→	flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)")
   127→	flag.BoolVar(&flagDev, "dev", false, "Enable development component: Swagger UI")
   128→	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
   129→	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
   130→	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
   131→	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info, warn (default), err, crit]`")
   132→	flag.Parse()
   133→
   134→	if flagVersion {
   135→		printVersion()
   136→		return nil
   137→	}
   138→
   139→	cclog.Init(flagLogLevel, flagLogDateTime)
   140→
   141→	if flagGops || config.Keys.Debug.EnableGops {
   142→		if err := agent.Listen(agent.Options{}); err != nil {
   143→			return fmt.Errorf("starting gops agent: %w", err)
   144→		}
   145→	}
   146→
   147→	ccconf.Init(flagConfigFile)
   148→
   149→	cfg := ccconf.GetPackageConfig("main")
   150→	if cfg == nil {
   151→		return fmt.Errorf("main configuration must be present")
   152→	}
   153→
   154→	config.Init(cfg)
   155→
   156→	natsConfig := ccconf.GetPackageConfig("nats")
   157→	if err := nats.Init(natsConfig); err != nil {
   158→		cclog.Warnf("initializing (optional) nats client: %s", err.Error())
   159→	}
   160→	nats.Connect()
   161→
   162→	ctx, cancel := context.WithCancel(context.Background())
   163→	defer cancel()
   164→
   165→	return runServer(ctx)
   166→}
   167→
   168→func main() {
   169→	if err := run(); err != nil {
   170→		cclog.Error(err.Error())
   171→		os.Exit(1)
   172→	}
   173→}
   174→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-metric-store.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"runtime/debug"
	"sync"
	"syscall"

	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/runtime"
	"github.com/ClusterCockpit/cc-metric-store/internal/api"
	"github.com/ClusterCockpit/cc-metric-store/internal/config"
	"github.com/google/gops/agent"
)

const (
	envGOGC = "GOGC"
)

var (
	date    string
	commit  string
	version string
)

var (
	flagGops, flagVersion, flagDev, flagLogDateTime bool
	flagConfigFile, flagLogLevel                    string
)

func printVersion() {
	fmt.Printf("Version:\t%s\n", version)
	fmt.Printf("Git hash:\t%s\n", commit)
	fmt.Printf("Build time:\t%s\n", date)
}

func runServer(ctx context.Context) error {
	var wg sync.WaitGroup

	mscfg := ccconf.GetPackageConfig("metrics")
	if mscfg == nil {
		return fmt.Errorf("missing metrics configuration")
	}
	config.InitMetrics(mscfg)

	mscfg = ccconf.GetPackageConfig("metric-store")
	if mscfg == nil {
		return fmt.Errorf("missing metricstore configuration")
	}

	metricstore.Init(mscfg, config.GetMetrics(), &wg)

	// Set GC percent if not configured
	if os.Getenv(envGOGC) == "" {
		debug.SetGCPercent(15)
	}

	if config.Keys.BackendURL != "" {
		ms := metricstore.GetMemoryStore()
		ms.SetNodeProvider(api.NewBackendNodeProvider(config.Keys.BackendURL))
		cclog.Infof("Node provider configured with backend URL: %s", config.Keys.BackendURL)
	}

	// Initialize HTTP server
	srv, err := NewServer(version, commit, date)
	if err != nil {
		return fmt.Errorf("creating server: %w", err)
	}

	// Channel to collect errors from server
	errChan := make(chan error, 1)

	// Start HTTP server
	wg.Go(func() {
		if err := srv.Start(ctx); err != nil {
			errChan <- err
		}
	})

	// Handle shutdown signals
	wg.Add(1)
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		defer wg.Done()
		select {
		case <-sigs:
			cclog.Info("Shutdown signal received")
		case <-ctx.Done():
		}

		runtime.SystemdNotify(false, "Shutting down ...")
		srv.Shutdown(ctx)
	}()

	runtime.SystemdNotify(true, "running")

	// Wait for completion or errors
	go func() {
		wg.Wait()
		close(errChan)
	}()

	// Wait for either server startup error or shutdown completion
	if err := <-errChan; err != nil {
		return err
	}

	cclog.Print("Graceful shutdown completed!")
	return nil
}

func run() error {
	flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)")
	flag.BoolVar(&flagDev, "dev", false, "Enable development component: Swagger UI")
	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info, warn (default), err, crit]`")
	flag.Parse()

	if flagVersion {
		printVersion()
		return nil
	}

	cclog.Init(flagLogLevel, flagLogDateTime)

	if flagGops || config.Keys.Debug.EnableGops {
		if err := agent.Listen(agent.Options{}); err != nil {
			return fmt.Errorf("starting gops agent: %w", err)
		}
	}

	ccconf.Init(flagConfigFile)

	cfg := ccconf.GetPackageConfig("main")
	if cfg == nil {
		return fmt.Errorf("main configuration must be present")
	}

	config.Init(cfg)

	natsConfig := ccconf.GetPackageConfig("nats")
	if err := nats.Init(natsConfig); err != nil {
		cclog.Warnf("initializing (optional) nats client: %s", err.Error())
	}
	nats.Connect()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	return runServer(ctx)
}

func main() {
	if err := run(); err != nil {
		cclog.Error(err.Error())
		os.Exit(1)
	}
}