     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
     7→//
     8→// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
     9→// per host, eliminating the extra memory overhead of the AvroStore and providing
    10→// truly continuous (per-write) crash safety.
    11→//
    12→// # Architecture
    13→//
    14→//	Metric write (DecodeLine)
    15→//	      │
    16→//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
    17→//	      │
    18→//	      └─► WALMessages channel
    19→//	               │
    20→//	               ▼
    21→//	        WALStaging goroutine
    22→//	               │
    23→//	               ▼
    24→//	        checkpoints/cluster/host/current.wal  (append-only, binary)
    25→//
    26→//	Periodic checkpoint (Checkpointing goroutine):
    27→//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
    28→//	  2. Signal WALStaging to truncate current.wal per host
    29→//
    30→//	On restart (FromCheckpoint):
    31→//	  1. Load most recent <timestamp>.bin snapshot
    32→//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
    33→//
    34→// # WAL Record Format
    35→//
    36→//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
    37→//
    38→//	payload:
    39→//	  [8B timestamp int64]
    40→//	  [2B metric_name_len uint16][N metric name bytes]
    41→//	  [1B selector_count uint8]
    42→//	  per selector: [1B selector_len uint8][M selector bytes]
    43→//	  [4B value float32 bits]
    44→//
    45→// # Binary Snapshot Format
    46→//
    47→//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
    48→//	Level tree (recursive):
    49→//	  [4B num_metrics uint32]
    50→//	  per metric:
    51→//	    [2B name_len uint16][N name bytes]
    52→//	    [8B frequency int64][8B start int64]
    53→//	    [4B num_values uint32][num_values × 4B float32]
    54→//	  [4B num_children uint32]
    55→//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
    56→package metricstore
    57→
    58→import (
    59→	"bufio"
    60→	"context"
    61→	"encoding/binary"
    62→	"fmt"
    63→	"hash/crc32"
    64→	"hash/fnv"
    65→	"io"
    66→	"math"
    67→	"os"
    68→	"path"
    69→	"strings"
    70→	"sync"
    71→	"sync/atomic"
    72→	"time"
    73→
    74→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    75→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    76→)
    77→
    78→// Magic numbers for binary formats.
    79→const (
    80→	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
    81→	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
    82→	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
    83→)
    84→
    85→// walShardChs holds per-shard channels for WAL messages.
    86→// Initialized by WALStaging; nil when WAL is not active.
    87→var walShardChs []chan *WALMessage
    88→
    89→// walShardRotateChs holds per-shard channels for WAL rotation requests.
    90→var walShardRotateChs []chan walRotateReq
    91→
    92→// walNumShards stores the number of shards (set during WALStaging init).
    93→var walNumShards int
    94→
    95→// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
    96→var walStagingWg sync.WaitGroup
    97→
    98→// walShuttingDown is set before closing shard channels to prevent
    99→// SendWALMessage from sending on a closed channel (which panics in Go).
   100→var walShuttingDown atomic.Bool
   101→
   102→// walCheckpointActive is set during binary checkpoint writes.
   103→// While active, SendWALMessage skips sending (returns true) because the
   104→// snapshot captures all in-memory data, making WAL writes redundant.
   105→var walCheckpointActive atomic.Bool
   106→
   107→// WALMessage represents a single metric write to be appended to the WAL.
   108→// Cluster and Node are NOT stored in the WAL record (inferred from file path).
   109→type WALMessage struct {
   110→	MetricName string
   111→	Cluster    string
   112→	Node       string
   113→	Selector   []string
   114→	Value      schema.Float
   115→	Timestamp  int64
   116→}
   117→
   118→// walRotateReq requests WAL file rotation for a specific host directory.
   119→// The done channel is closed by the WAL goroutine when rotation is complete.
   120→type walRotateReq struct {
   121→	hostDir string
   122→	done    chan struct{}
   123→}
   124→
   125→// walFileState holds an open WAL file handle and buffered writer for one host directory.
   126→type walFileState struct {
   127→	f     *os.File
   128→	w     *bufio.Writer
   129→	dirty bool
   130→	size  int64 // approximate bytes written (tracked from open + writes)
   131→}
   132→
   133→// walFlushInterval controls how often dirty WAL files are flushed to disk.
   134→// Decoupling flushes from message processing lets the consumer run at memory
   135→// speed, amortizing syscall overhead across many writes.
   136→const walFlushInterval = 1 * time.Second
   137→
   138→// walShardIndex computes which shard a message belongs to based on cluster+node.
   139→// Uses FNV-1a hash for fast, well-distributed mapping.
   140→func walShardIndex(cluster, node string) int {
   141→	h := fnv.New32a()
   142→	h.Write([]byte(cluster))
   143→	h.Write([]byte{0})
   144→	h.Write([]byte(node))
   145→	return int(h.Sum32()) % walNumShards
   146→}
   147→
   148→// SendWALMessage routes a WAL message to the appropriate shard channel.
   149→// Returns false if the channel is full or shutdown is in progress.
   150→func SendWALMessage(msg *WALMessage) bool {
   151→	if walShardChs == nil || walShuttingDown.Load() {
   152→		return false
   153→	}
   154→	if walCheckpointActive.Load() {
   155→		return true // Data safe in memory; snapshot will capture it
   156→	}
   157→	shard := walShardIndex(msg.Cluster, msg.Node)
   158→	select {
   159→	case walShardChs[shard] <- msg:
   160→		return true
   161→	default:
   162→		return false
   163→	}
   164→}
   165→
   166→// WALStaging starts N sharded background goroutines that receive WALMessage items
   167→// and append binary WAL records to per-host current.wal files.
   168→// Each shard owns a subset of hosts (determined by hash of cluster+node),
   169→// parallelizing serialization and I/O while keeping per-host writes serial.
   170→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   171→	if Keys.Checkpoints.FileFormat == "json" {
   172→		return
   173→	}
   174→
   175→	walNumShards = max(Keys.NumWorkers, 1)
   176→	chBufSize := max(65536/walNumShards, 1024)
   177→
   178→	walShardChs = make([]chan *WALMessage, walNumShards)
   179→	walShardRotateChs = make([]chan walRotateReq, walNumShards)
   180→
   181→	for i := range walNumShards {
   182→		walShardChs[i] = make(chan *WALMessage, chBufSize)
   183→		walShardRotateChs[i] = make(chan walRotateReq, 64)
   184→	}
   185→
   186→	for i := range walNumShards {
   187→		msgCh := walShardChs[i]
   188→		rotateCh := walShardRotateChs[i]
   189→
   190→		walStagingWg.Add(1)
   191→		wg.Go(func() {
   192→			defer walStagingWg.Done()
   193→			hostFiles := make(map[string]*walFileState)
   194→
   195→			defer func() {
   196→				for _, ws := range hostFiles {
   197→					if ws.f != nil {
   198→						ws.w.Flush()
   199→						ws.f.Close()
   200→					}
   201→				}
   202→			}()
   203→
   204→			getOrOpenWAL := func(hostDir string) *walFileState {
   205→				ws, ok := hostFiles[hostDir]
   206→				if ok {
   207→					return ws
   208→				}
   209→
   210→				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   211→					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   212→					return nil
   213→				}
   214→
   215→				walPath := path.Join(hostDir, "current.wal")
   216→				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   217→				if err != nil {
   218→					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   219→					return nil
   220→				}
   221→
   222→				w := bufio.NewWriter(f)
   223→
   224→				// Write file header magic if file is new (empty).
   225→				info, err := f.Stat()
   226→				var fileSize int64
   227→				if err == nil {
   228→					fileSize = info.Size()
   229→				}
   230→				if err == nil && fileSize == 0 {
   231→					var hdr [4]byte
   232→					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   233→					if _, err := w.Write(hdr[:]); err != nil {
   234→						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   235→						f.Close()
   236→						return nil
   237→					}
   238→					fileSize = 4
   239→				}
   240→
   241→				ws = &walFileState{f: f, w: w, size: fileSize}
   242→				hostFiles[hostDir] = ws
   243→				return ws
   244→			}
   245→
   246→			processMsg := func(msg *WALMessage) {
   247→				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   248→				ws := getOrOpenWAL(hostDir)
   249→				if ws == nil {
   250→					return
   251→				}
   252→
   253→				// Enforce max WAL size: force-rotate before writing if limit is exceeded.
   254→				// The in-memory store still holds the data; only crash-recovery coverage is lost.
   255→				if maxSize := Keys.Checkpoints.MaxWALSize; maxSize > 0 && ws.size >= maxSize {
   256→					cclog.Warnf("[METRICSTORE]> WAL: force-rotating %s (size %d >= limit %d)",
   257→						hostDir, ws.size, maxSize)
   258→					ws.w.Flush()
   259→					ws.f.Close()
   260→					walPath := path.Join(hostDir, "current.wal")
   261→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   262→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   263→					}
   264→					delete(hostFiles, hostDir)
   265→					ws = getOrOpenWAL(hostDir)
   266→					if ws == nil {
   267→						return
   268→					}
   269→				}
   270→
   271→				n, err := writeWALRecordDirect(ws.w, msg)
   272→				if err != nil {
   273→					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   274→				}
   275→				ws.size += int64(n)
   276→				ws.dirty = true
   277→			}
   278→
   279→			processRotate := func(req walRotateReq) {
   280→				ws, ok := hostFiles[req.hostDir]
   281→				if ok && ws.f != nil {
   282→					ws.w.Flush()
   283→					ws.f.Close()
   284→					walPath := path.Join(req.hostDir, "current.wal")
   285→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   286→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   287→					}
   288→					delete(hostFiles, req.hostDir)
   289→				}
   290→				close(req.done)
   291→			}
   292→
   293→			flushDirty := func() {
   294→				for _, ws := range hostFiles {
   295→					if ws.dirty {
   296→						ws.w.Flush()
   297→						ws.dirty = false
   298→					}
   299→				}
   300→			}
   301→
   302→			ticker := time.NewTicker(walFlushInterval)
   303→			defer ticker.Stop()
   304→
   305→			// drainBatch processes up to 4096 pending messages without blocking.
   306→			// Returns false if the channel was closed.
   307→			drainBatch := func() bool {
   308→				for range 4096 {
   309→					select {
   310→					case msg, ok := <-msgCh:
   311→						if !ok {
   312→							flushDirty()
   313→							return false
   314→						}
   315→						processMsg(msg)
   316→					case req := <-rotateCh:
   317→						processRotate(req)
   318→					default:
   319→						return true
   320→					}
   321→				}
   322→				return true
   323→			}
   324→
   325→			for {
   326→				select {
   327→				case <-ctx.Done():
   328→					// On shutdown, skip draining buffered messages — a full binary
   329→					// checkpoint will be written from in-memory state, making
   330→					// buffered WAL records redundant.
   331→					flushDirty()
   332→					return
   333→				case msg, ok := <-msgCh:
   334→					if !ok {
   335→						return
   336→					}
   337→					processMsg(msg)
   338→					if !drainBatch() {
   339→						return
   340→					}
   341→					// No flush here — timer handles periodic flushing.
   342→				case <-ticker.C:
   343→					flushDirty()
   344→				case req := <-rotateCh:
   345→					processRotate(req)
   346→				}
   347→			}
   348→		})
   349→	}
   350→}
   351→
   352→// WaitForWALStagingDrain blocks until all WALStaging goroutines have exited.
   353→// Must be called after closing walShardChs to ensure all file handles are
   354→// flushed and closed before checkpoint writes begin.
   355→func WaitForWALStagingDrain() {
   356→	walStagingWg.Wait()
   357→}
   358→
   359→// RotateWALFiles sends rotation requests for the given host directories
   360→// and blocks until all rotations complete. Each request is routed to the
   361→// shard that owns the host directory.
   362→//
   363→// If shutdown is in progress (WAL staging goroutines may have exited),
   364→// rotation is skipped to avoid deadlocking on abandoned channels.
   365→func RotateWALFiles(hostDirs []string) {
   366→	if walShardRotateChs == nil || walShuttingDown.Load() {
   367→		return
   368→	}
   369→	deadline := time.After(2 * time.Minute)
   370→	dones := make([]chan struct{}, 0, len(hostDirs))
   371→	for _, dir := range hostDirs {
   372→		done := make(chan struct{})
   373→		shard := walShardIndexFromDir(dir)
   374→		select {
   375→		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
   376→			dones = append(dones, done)
   377→		case <-deadline:
   378→			cclog.Warnf("[METRICSTORE]> WAL rotation send timed out, %d of %d hosts remaining",
   379→				len(hostDirs)-len(dones), len(hostDirs))
   380→			goto waitDones
   381→		}
   382→	}
   383→waitDones:
   384→	for _, done := range dones {
   385→		select {
   386→		case <-done:
   387→		case <-time.After(30 * time.Second):
   388→			cclog.Warn("[METRICSTORE]> WAL rotation completion timed out, continuing")
   389→			return
   390→		}
   391→	}
   392→}
   393→
   394→// walShardIndexFromDir extracts cluster and node from a host directory path
   395→// and returns the shard index. The path format is: rootDir/cluster/node
   396→func walShardIndexFromDir(hostDir string) int {
   397→	// hostDir = .../cluster/node — extract last two path components
   398→	node := path.Base(hostDir)
   399→	cluster := path.Base(path.Dir(hostDir))
   400→	return walShardIndex(cluster, node)
   401→}
   402→
   403→// RotateWALFiles sends rotation requests for the given host directories
   404→// and blocks until all rotations complete.
   405→func RotateWALFilesAfterShutdown(hostDirs []string) {
   406→	for _, dir := range hostDirs {
   407→		walPath := path.Join(dir, "current.wal")
   408→		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   409→			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   410→		}
   411→	}
   412→}
   413→
   414→// writeWALRecordDirect encodes a WAL record into a contiguous buffer first,
   415→// then writes it to the bufio.Writer in a single call. This prevents partial
   416→// records in the write buffer if a write error occurs mid-record (e.g. disk full).
   417→// Returns the number of bytes written and any error.
   418→func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) (int, error) {
   419→	// Compute payload and total record size.
   420→	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
   421→	for _, s := range msg.Selector {
   422→		payloadSize += 1 + len(s)
   423→	}
   424→	// Total: 8 (header) + payload + 4 (CRC).
   425→	totalSize := 8 + payloadSize + 4
   426→
   427→	// Use stack buffer for typical small records, heap-allocate only for large ones.
   428→	var stackBuf [256]byte
   429→	var buf []byte
   430→	if totalSize <= len(stackBuf) {
   431→		buf = stackBuf[:totalSize]
   432→	} else {
   433→		buf = make([]byte, totalSize)
   434→	}
   435→
   436→	// Header: magic + payload length.
   437→	binary.LittleEndian.PutUint32(buf[0:4], walRecordMagic)
   438→	binary.LittleEndian.PutUint32(buf[4:8], uint32(payloadSize))
   439→
   440→	// Payload starts at offset 8.
   441→	p := 8
   442→
   443→	// Timestamp (8 bytes).
   444→	binary.LittleEndian.PutUint64(buf[p:p+8], uint64(msg.Timestamp))
   445→	p += 8
   446→
   447→	// Metric name length (2 bytes) + metric name.
   448→	binary.LittleEndian.PutUint16(buf[p:p+2], uint16(len(msg.MetricName)))
   449→	p += 2
   450→	p += copy(buf[p:], msg.MetricName)
   451→
   452→	// Selector count (1 byte).
   453→	buf[p] = byte(len(msg.Selector))
   454→	p++
   455→
   456→	// Selectors (1-byte length + bytes each).
   457→	for _, sel := range msg.Selector {
   458→		buf[p] = byte(len(sel))
   459→		p++
   460→		p += copy(buf[p:], sel)
   461→	}
   462→
   463→	// Value (4 bytes, float32 bits).
   464→	binary.LittleEndian.PutUint32(buf[p:p+4], math.Float32bits(float32(msg.Value)))
   465→	p += 4
   466→
   467→	// CRC32 over payload (bytes 8..8+payloadSize).
   468→	crc := crc32.ChecksumIEEE(buf[8 : 8+payloadSize])
   469→	binary.LittleEndian.PutUint32(buf[p:p+4], crc)
   470→
   471→	// Single atomic write to the buffered writer.
   472→	n, err := w.Write(buf)
   473→	return n, err
   474→}
   475→
   476→// readWALRecord reads one WAL record from the reader.
   477→// Returns (nil, nil) on clean EOF. Returns error on data corruption.
   478→// A CRC mismatch indicates a truncated trailing record (expected on crash).
   479→func readWALRecord(r io.Reader) (*WALMessage, error) {
   480→	var magic uint32
   481→	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
   482→		if err == io.EOF {
   483→			return nil, nil // Clean EOF
   484→		}
   485→		return nil, fmt.Errorf("read record magic: %w", err)
   486→	}
   487→
   488→	if magic != walRecordMagic {
   489→		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
   490→	}
   491→
   492→	var payloadLen uint32
   493→	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
   494→		return nil, fmt.Errorf("read payload length: %w", err)
   495→	}
   496→
   497→	if payloadLen > 1<<20 { // 1 MB sanity limit
   498→		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
   499→	}
   500→
   501→	payload := make([]byte, payloadLen)
   502→	if _, err := io.ReadFull(r, payload); err != nil {
   503→		return nil, fmt.Errorf("read payload: %w", err)
   504→	}
   505→
   506→	var storedCRC uint32
   507→	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
   508→		return nil, fmt.Errorf("read CRC: %w", err)
   509→	}
   510→
   511→	if crc32.ChecksumIEEE(payload) != storedCRC {
   512→		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
   513→	}
   514→
   515→	return parseWALPayload(payload)
   516→}
   517→
   518→// parseWALPayload decodes a binary payload into a WALMessage.
   519→func parseWALPayload(payload []byte) (*WALMessage, error) {
   520→	if len(payload) < 8+2+1+4 {
   521→		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
   522→	}
   523→
   524→	offset := 0
   525→
   526→	// Timestamp (8 bytes)
   527→	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
   528→	offset += 8
   529→
   530→	// Metric name (2-byte length + bytes)
   531→	if offset+2 > len(payload) {
   532→		return nil, fmt.Errorf("metric name length overflows payload")
   533→	}
   534→	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
   535→	offset += 2
   536→
   537→	if offset+mLen > len(payload) {
   538→		return nil, fmt.Errorf("metric name overflows payload")
   539→	}
   540→	metricName := string(payload[offset : offset+mLen])
   541→	offset += mLen
   542→
   543→	// Selector count (1 byte)
   544→	if offset >= len(payload) {
   545→		return nil, fmt.Errorf("selector count overflows payload")
   546→	}
   547→	selCount := int(payload[offset])
   548→	offset++
   549→
   550→	selectors := make([]string, selCount)
   551→	for i := range selCount {
   552→		if offset >= len(payload) {
   553→			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
   554→		}
   555→		sLen := int(payload[offset])
   556→		offset++
   557→
   558→		if offset+sLen > len(payload) {
   559→			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
   560→		}
   561→		selectors[i] = string(payload[offset : offset+sLen])
   562→		offset += sLen
   563→	}
   564→
   565→	// Value (4 bytes, float32 bits)
   566→	if offset+4 > len(payload) {
   567→		return nil, fmt.Errorf("value overflows payload")
   568→	}
   569→	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
   570→	value := schema.Float(math.Float32frombits(bits))
   571→
   572→	return &WALMessage{
   573→		MetricName: metricName,
   574→		Timestamp:  ts,
   575→		Selector:   selectors,
   576→		Value:      value,
   577→	}, nil
   578→}
   579→
   580→// loadWALFile reads a WAL file and replays all valid records into the Level tree.
   581→// l is the host-level node. Corrupt or partial trailing records are silently skipped
   582→// (expected on crash). Records older than 'from' are skipped.
   583→func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
   584→	br := bufio.NewReader(f)
   585→
   586→	// Verify file header magic.
   587→	var fileMagic uint32
   588→	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
   589→		if err == io.EOF {
   590→			return nil // Empty file, no data
   591→		}
   592→		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
   593→	}
   594→
   595→	if fileMagic != walFileMagic {
   596→		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
   597→	}
   598→
   599→	// Cache level lookups to avoid repeated tree traversal.
   600→	lvlCache := make(map[string]*Level)
   601→
   602→	for {
   603→		msg, err := readWALRecord(br)
   604→		if err != nil {
   605→			// Truncated trailing record is expected after a crash; stop replaying.
   606→			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
   607→			break
   608→		}
   609→		if msg == nil {
   610→			break // Clean EOF
   611→		}
   612→
   613→		if msg.Timestamp < from {
   614→			continue // Older than retention window
   615→		}
   616→
   617→		minfo, ok := m.Metrics[msg.MetricName]
   618→		if !ok {
   619→			continue // Unknown metric (config may have changed)
   620→		}
   621→
   622→		// Cache key is the null-separated selector path.
   623→		cacheKey := joinSelector(msg.Selector)
   624→		lvl, ok := lvlCache[cacheKey]
   625→		if !ok {
   626→			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
   627→			lvlCache[cacheKey] = lvl
   628→		}
   629→
   630→		// Write directly to the buffer, same as WriteToLevel but without the
   631→		// global level lookup (we already have the right level).
   632→		lvl.lock.Lock()
   633→		b := lvl.metrics[minfo.offset]
   634→		if b == nil {
   635→			b = newBuffer(msg.Timestamp, minfo.Frequency)
   636→			lvl.metrics[minfo.offset] = b
   637→		}
   638→		nb, writeErr := b.write(msg.Timestamp, msg.Value)
   639→		if writeErr == nil && b != nb {
   640→			lvl.metrics[minfo.offset] = nb
   641→		}
   642→		// Ignore write errors for timestamps before buffer start (can happen when
   643→		// replaying WAL entries that predate a loaded snapshot's start time).
   644→		lvl.lock.Unlock()
   645→	}
   646→
   647→	return nil
   648→}
   649→
   650→// joinSelector builds a cache key from a selector slice using null bytes as separators.
   651→func joinSelector(sel []string) string {
   652→	if len(sel) == 0 {
   653→		return ""
   654→	}
   655→	var result strings.Builder
   656→	result.WriteString(sel[0])
   657→	for i := 1; i < len(sel); i++ {
   658→		result.WriteString("\x00" + sel[i])
   659→	}
   660→	return result.String()
   661→}
   662→
   663→// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
   664→// Returns the number of files written, the list of host directories that were
   665→// successfully checkpointed (for WAL rotation), and any errors.
   666→func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
   667→	// Collect all cluster/host pairs.
   668→	m.root.lock.RLock()
   669→	totalHosts := 0
   670→	for _, l1 := range m.root.children {
   671→		l1.lock.RLock()
   672→		totalHosts += len(l1.children)
   673→		l1.lock.RUnlock()
   674→	}
   675→	m.root.lock.RUnlock()
   676→
   677→	levels := make([]*Level, 0, totalHosts)
   678→	selectors := make([][]string, 0, totalHosts)
   679→
   680→	m.root.lock.RLock()
   681→	for sel1, l1 := range m.root.children {
   682→		l1.lock.RLock()
   683→		for sel2, l2 := range l1.children {
   684→			levels = append(levels, l2)
   685→			selectors = append(selectors, []string{sel1, sel2})
   686→		}
   687→		l1.lock.RUnlock()
   688→	}
   689→	m.root.lock.RUnlock()
   690→
   691→	type workItem struct {
   692→		level    *Level
   693→		hostDir  string
   694→		selector []string
   695→	}
   696→
   697→	totalWork := len(levels)
   698→	cclog.Infof("[METRICSTORE]> Starting binary checkpoint for %d hosts with %d workers", totalWork, Keys.NumWorkers)
   699→
   700→	n, errs, completed := int32(0), int32(0), int32(0)
   701→	var successDirs []string
   702→	var successMu sync.Mutex
   703→
   704→	var wg sync.WaitGroup
   705→	wg.Add(Keys.NumWorkers)
   706→	work := make(chan workItem, Keys.NumWorkers*2)
   707→
   708→	// Progress logging goroutine.
   709→	stopProgress := make(chan struct{})
   710→	go func() {
   711→		ticker := time.NewTicker(10 * time.Second)
   712→		defer ticker.Stop()
   713→		for {
   714→			select {
   715→			case <-ticker.C:
   716→				cclog.Infof("[METRICSTORE]> Checkpoint progress: %d/%d hosts (%d written, %d errors)",
   717→					atomic.LoadInt32(&completed), totalWork, atomic.LoadInt32(&n), atomic.LoadInt32(&errs))
   718→			case <-stopProgress:
   719→				return
   720→			}
   721→		}
   722→	}()
   723→
   724→	for range Keys.NumWorkers {
   725→		go func() {
   726→			defer wg.Done()
   727→			for wi := range work {
   728→				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
   729→				if err != nil {
   730→					if err == ErrNoNewArchiveData {
   731→						atomic.AddInt32(&completed, 1)
   732→						continue
   733→					}
   734→					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
   735→					atomic.AddInt32(&errs, 1)
   736→				} else {
   737→					atomic.AddInt32(&n, 1)
   738→					successMu.Lock()
   739→					successDirs = append(successDirs, wi.hostDir)
   740→					successMu.Unlock()
   741→				}
   742→				atomic.AddInt32(&completed, 1)
   743→			}
   744→		}()
   745→	}
   746→
   747→	for i := range levels {
   748→		hostDir := path.Join(dir, path.Join(selectors[i]...))
   749→		work <- workItem{
   750→			level:    levels[i],
   751→			hostDir:  hostDir,
   752→			selector: selectors[i],
   753→		}
   754→	}
   755→	close(work)
   756→	wg.Wait()
   757→	close(stopProgress)
   758→
   759→	if errs > 0 {
   760→		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
   761→	}
   762→	return int(n), successDirs, nil
   763→}
   764→
   765→// toCheckpointBinary writes a binary snapshot file for a single host-level node.
   766→// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
   767→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   768→	cf, err := l.toCheckpointFile(from, to, m)
   769→	if err != nil {
   770→		return err
   771→	}
   772→	if cf == nil {
   773→		return ErrNoNewArchiveData
   774→	}
   775→
   776→	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
   777→		return fmt.Errorf("mkdir %s: %w", dir, err)
   778→	}
   779→
   780→	// Write to a temp file first, then rename (atomic on POSIX).
   781→	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
   782→	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))
   783→
   784→	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   785→	if err != nil {
   786→		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
   787→	}
   788→
   789→	bw := bufio.NewWriter(f)
   790→	if err := writeBinarySnapshotFile(bw, cf); err != nil {
   791→		f.Close()
   792→		os.Remove(tmpPath)
   793→		return fmt.Errorf("write binary snapshot: %w", err)
   794→	}
   795→	if err := bw.Flush(); err != nil {
   796→		f.Close()
   797→		os.Remove(tmpPath)
   798→		return err
   799→	}
   800→	f.Close()
   801→
   802→	return os.Rename(tmpPath, finalPath)
   803→}
   804→
   805→// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
   806→func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
   807→	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
   808→		return err
   809→	}
   810→	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
   811→		return err
   812→	}
   813→	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
   814→		return err
   815→	}
   816→	return writeBinaryLevel(w, cf)
   817→}
   818→
   819→// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
   820→func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
   821→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
   822→		return err
   823→	}
   824→
   825→	for name, metric := range cf.Metrics {
   826→		if err := writeString16(w, name); err != nil {
   827→			return err
   828→		}
   829→		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
   830→			return err
   831→		}
   832→		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
   833→			return err
   834→		}
   835→		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
   836→			return err
   837→		}
   838→		for _, v := range metric.Data {
   839→			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
   840→				return err
   841→			}
   842→		}
   843→	}
   844→
   845→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
   846→		return err
   847→	}
   848→
   849→	for name, child := range cf.Children {
   850→		if err := writeString16(w, name); err != nil {
   851→			return err
   852→		}
   853→		if err := writeBinaryLevel(w, child); err != nil {
   854→			return err
   855→		}
   856→	}
   857→
   858→	return nil
   859→}
   860→
   861→// writeString16 writes a 2-byte length-prefixed string to w.
   862→func writeString16(w io.Writer, s string) error {
   863→	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
   864→		return err
   865→	}
   866→	_, err := io.WriteString(w, s)
   867→	return err
   868→}
   869→
   870→// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
   871→// The retention check (from) is applied to the file's 'to' timestamp.
   872→func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
   873→	br := bufio.NewReader(f)
   874→
   875→	var magic uint32
   876→	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
   877→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
   878→	}
   879→	if magic != snapFileMagic {
   880→		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
   881→	}
   882→
   883→	var fileFrom, fileTo int64
   884→	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
   885→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
   886→	}
   887→	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
   888→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
   889→	}
   890→
   891→	if fileTo != 0 && fileTo < from {
   892→		return nil // File is older than retention window, skip it
   893→	}
   894→
   895→	cf, err := readBinaryLevel(br)
   896→	if err != nil {
   897→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
   898→	}
   899→	cf.From = fileFrom
   900→	cf.To = fileTo
   901→
   902→	return l.loadFile(cf, m)
   903→}
   904→
   905→// readBinaryLevel recursively reads a level from the binary snapshot format.
   906→func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
   907→	cf := &CheckpointFile{
   908→		Metrics:  make(map[string]*CheckpointMetrics),
   909→		Children: make(map[string]*CheckpointFile),
   910→	}
   911→
   912→	var numMetrics uint32
   913→	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
   914→		return nil, fmt.Errorf("read num_metrics: %w", err)
   915→	}
   916→
   917→	for range numMetrics {
   918→		name, err := readString16(r)
   919→		if err != nil {
   920→			return nil, fmt.Errorf("read metric name: %w", err)
   921→		}
   922→
   923→		var freq, start int64
   924→		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
   925→			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
   926→		}
   927→		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
   928→			return nil, fmt.Errorf("read start for %s: %w", name, err)
   929→		}
   930→
   931→		var numValues uint32
   932→		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
   933→			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
   934→		}
   935→
   936→		data := make([]schema.Float, numValues)
   937→		for i := range numValues {
   938→			var bits uint32
   939→			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
   940→				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
   941→			}
   942→			data[i] = schema.Float(math.Float32frombits(bits))
   943→		}
   944→
   945→		cf.Metrics[name] = &CheckpointMetrics{
   946→			Frequency: freq,
   947→			Start:     start,
   948→			Data:      data,
   949→		}
   950→	}
   951→
   952→	var numChildren uint32
   953→	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
   954→		return nil, fmt.Errorf("read num_children: %w", err)
   955→	}
   956→
   957→	for range numChildren {
   958→		childName, err := readString16(r)
   959→		if err != nil {
   960→			return nil, fmt.Errorf("read child name: %w", err)
   961→		}
   962→
   963→		child, err := readBinaryLevel(r)
   964→		if err != nil {
   965→			return nil, fmt.Errorf("read child %s: %w", childName, err)
   966→		}
   967→		cf.Children[childName] = child
   968→	}
   969→
   970→	return cf, nil
   971→}
   972→
   973→// readString16 reads a 2-byte length-prefixed string from r.
   974→func readString16(r io.Reader) (string, error) {
   975→	var sLen uint16
   976→	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
   977→		return "", err
   978→	}
   979→	buf := make([]byte, sLen)
   980→	if _, err := io.ReadFull(r, buf); err != nil {
   981→		return "", err
   982→	}
   983→	return string(buf), nil
   984→}
   985→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼
//	        WALStaging goroutine
//	               │
//	               ▼
//	        checkpoints/cluster/host/current.wal  (append-only, binary)
//
//	Periodic checkpoint (Checkpointing goroutine):
//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
//	  2. Signal WALStaging to truncate current.wal per host
//
//	On restart (FromCheckpoint):
//	  1. Load most recent <timestamp>.bin snapshot
//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
//
// # WAL Record Format
//
//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
//
//	payload:
//	  [8B timestamp int64]
//	  [2B metric_name_len uint16][N metric name bytes]
//	  [1B selector_count uint8]
//	  per selector: [1B selector_len uint8][M selector bytes]
//	  [4B value float32 bits]
//
// # Binary Snapshot Format
//
//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
//	Level tree (recursive):
//	  [4B num_metrics uint32]
//	  per metric:
//	    [2B name_len uint16][N name bytes]
//	    [8B frequency int64][8B start int64]
//	    [4B num_values uint32][num_values × 4B float32]
//	  [4B num_children uint32]
//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
package metricstore

import (
	"bufio"
	"context"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"hash/fnv"
	"io"
	"math"
	"os"
	"path"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Magic numbers for binary formats.
const (
	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
)

// walShardChs holds per-shard channels for WAL messages.
// Initialized by WALStaging; nil when WAL is not active.
var walShardChs []chan *WALMessage

// walShardRotateChs holds per-shard channels for WAL rotation requests.
var walShardRotateChs []chan walRotateReq

// walNumShards stores the number of shards (set during WALStaging init).
var walNumShards int

// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
var walStagingWg sync.WaitGroup

// walShuttingDown is set before closing shard channels to prevent
// SendWALMessage from sending on a closed channel (which panics in Go).
var walShuttingDown atomic.Bool

// walCheckpointActive is set during binary checkpoint writes.
// While active, SendWALMessage skips sending (returns true) because the
// snapshot captures all in-memory data, making WAL writes redundant.
var walCheckpointActive atomic.Bool

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f     *os.File
	w     *bufio.Writer
	dirty bool
	size  int64 // approximate bytes written (tracked from open + writes)
}

// walFlushInterval controls how often dirty WAL files are flushed to disk.
// Decoupling flushes from message processing lets the consumer run at memory
// speed, amortizing syscall overhead across many writes.
const walFlushInterval = 1 * time.Second

// walShardIndex computes which shard a message belongs to based on cluster+node.
// Uses FNV-1a hash for fast, well-distributed mapping.
func walShardIndex(cluster, node string) int {
	h := fnv.New32a()
	h.Write([]byte(cluster))
	h.Write([]byte{0})
	h.Write([]byte(node))
	return int(h.Sum32()) % walNumShards
}

// SendWALMessage routes a WAL message to the appropriate shard channel.
// Returns false if the channel is full or shutdown is in progress.
func SendWALMessage(msg *WALMessage) bool {
	if walShardChs == nil || walShuttingDown.Load() {
		return false
	}
	if walCheckpointActive.Load() {
		return true // Data safe in memory; snapshot will capture it
	}
	shard := walShardIndex(msg.Cluster, msg.Node)
	select {
	case walShardChs[shard] <- msg:
		return true
	default:
		return false
	}
}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.
// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return
	}

	walNumShards = max(Keys.NumWorkers, 1)
	chBufSize := max(65536/walNumShards, 1024)

	walShardChs = make([]chan *WALMessage, walNumShards)
	walShardRotateChs = make([]chan walRotateReq, walNumShards)

	for i := range walNumShards {
		walShardChs[i] = make(chan *WALMessage, chBufSize)
		walShardRotateChs[i] = make(chan walRotateReq, 64)
	}

	for i := range walNumShards {
		msgCh := walShardChs[i]
		rotateCh := walShardRotateChs[i]

		walStagingWg.Add(1)
		wg.Go(func() {
			defer walStagingWg.Done()
			hostFiles := make(map[string]*walFileState)

			defer func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
						ws.f.Close()
					}
				}
			}()

			getOrOpenWAL := func(hostDir string) *walFileState {
				ws, ok := hostFiles[hostDir]
				if ok {
					return ws
				}

				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
					return nil
				}

				walPath := path.Join(hostDir, "current.wal")
				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
					return nil
				}

				w := bufio.NewWriter(f)

				// Write file header magic if file is new (empty).
				info, err := f.Stat()
				var fileSize int64
				if err == nil {
					fileSize = info.Size()
				}
				if err == nil && fileSize == 0 {
					var hdr [4]byte
					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
					if _, err := w.Write(hdr[:]); err != nil {
						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
						f.Close()
						return nil
					}
					fileSize = 4
				}

				ws = &walFileState{f: f, w: w, size: fileSize}
				hostFiles[hostDir] = ws
				return ws
			}

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}

				// Enforce max WAL size: force-rotate before writing if limit is exceeded.
				// The in-memory store still holds the data; only crash-recovery coverage is lost.
				if maxSize := Keys.Checkpoints.MaxWALSize; maxSize > 0 && ws.size >= maxSize {
					cclog.Warnf("[METRICSTORE]> WAL: force-rotating %s (size %d >= limit %d)",
						hostDir, ws.size, maxSize)
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, hostDir)
					ws = getOrOpenWAL(hostDir)
					if ws == nil {
						return
					}
				}

				n, err := writeWALRecordDirect(ws.w, msg)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
				ws.size += int64(n)
				ws.dirty = true
			}

			processRotate := func(req walRotateReq) {
				ws, ok := hostFiles[req.hostDir]
				if ok && ws.f != nil {
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(req.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, req.hostDir)
				}
				close(req.done)
			}

			flushDirty := func() {
				for _, ws := range hostFiles {
					if ws.dirty {
						ws.w.Flush()
						ws.dirty = false
					}
				}
			}

			ticker := time.NewTicker(walFlushInterval)
			defer ticker.Stop()

			// drainBatch processes up to 4096 pending messages without blocking.
			// Returns false if the channel was closed.
			drainBatch := func() bool {
				for range 4096 {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							flushDirty()
							return false
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						return true
					}
				}
				return true
			}

			for {
				select {
				case <-ctx.Done():
					// On shutdown, skip draining buffered messages — a full binary
					// checkpoint will be written from in-memory state, making
					// buffered WAL records redundant.
					flushDirty()
					return
				case msg, ok := <-msgCh:
					if !ok {
						return
					}
					processMsg(msg)
					if !drainBatch() {
						return
					}
					// No flush here — timer handles periodic flushing.
				case <-ticker.C:
					flushDirty()
				case req := <-rotateCh:
					processRotate(req)
				}
			}
		})
	}
}

// WaitForWALStagingDrain blocks until all WALStaging goroutines have exited.
// Must be called after closing walShardChs to ensure all file handles are
// flushed and closed before checkpoint writes begin.
func WaitForWALStagingDrain() {
	walStagingWg.Wait()
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete. Each request is routed to the
// shard that owns the host directory.
//
// If shutdown is in progress (WAL staging goroutines may have exited),
// rotation is skipped to avoid deadlocking on abandoned channels.
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	deadline := time.After(2 * time.Minute)
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		case <-deadline:
			cclog.Warnf("[METRICSTORE]> WAL rotation send timed out, %d of %d hosts remaining",
				len(hostDirs)-len(dones), len(hostDirs))
			goto waitDones
		}
	}
waitDones:
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation completion timed out, continuing")
			return
		}
	}
}

// walShardIndexFromDir extracts cluster and node from a host directory path
// and returns the shard index. The path format is: rootDir/cluster/node
func walShardIndexFromDir(hostDir string) int {
	// hostDir = .../cluster/node — extract last two path components
	node := path.Base(hostDir)
	cluster := path.Base(path.Dir(hostDir))
	return walShardIndex(cluster, node)
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete.
func RotateWALFilesAfterShutdown(hostDirs []string) {
	for _, dir := range hostDirs {
		walPath := path.Join(dir, "current.wal")
		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
		}
	}
}

// writeWALRecordDirect encodes a WAL record into a contiguous buffer first,
// then writes it to the bufio.Writer in a single call. This prevents partial
// records in the write buffer if a write error occurs mid-record (e.g. disk full).
// Returns the number of bytes written and any error.
func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) (int, error) {
	// Compute payload and total record size.
	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
	for _, s := range msg.Selector {
		payloadSize += 1 + len(s)
	}
	// Total: 8 (header) + payload + 4 (CRC).
	totalSize := 8 + payloadSize + 4

	// Use stack buffer for typical small records, heap-allocate only for large ones.
	var stackBuf [256]byte
	var buf []byte
	if totalSize <= len(stackBuf) {
		buf = stackBuf[:totalSize]
	} else {
		buf = make([]byte, totalSize)
	}

	// Header: magic + payload length.
	binary.LittleEndian.PutUint32(buf[0:4], walRecordMagic)
	binary.LittleEndian.PutUint32(buf[4:8], uint32(payloadSize))

	// Payload starts at offset 8.
	p := 8

	// Timestamp (8 bytes).
	binary.LittleEndian.PutUint64(buf[p:p+8], uint64(msg.Timestamp))
	p += 8

	// Metric name length (2 bytes) + metric name.
	binary.LittleEndian.PutUint16(buf[p:p+2], uint16(len(msg.MetricName)))
	p += 2
	p += copy(buf[p:], msg.MetricName)

	// Selector count (1 byte).
	buf[p] = byte(len(msg.Selector))
	p++

	// Selectors (1-byte length + bytes each).
	for _, sel := range msg.Selector {
		buf[p] = byte(len(sel))
		p++
		p += copy(buf[p:], sel)
	}

	// Value (4 bytes, float32 bits).
	binary.LittleEndian.PutUint32(buf[p:p+4], math.Float32bits(float32(msg.Value)))
	p += 4

	// CRC32 over payload (bytes 8..8+payloadSize).
	crc := crc32.ChecksumIEEE(buf[8 : 8+payloadSize])
	binary.LittleEndian.PutUint32(buf[p:p+4], crc)

	// Single atomic write to the buffered writer.
	n, err := w.Write(buf)
	return n, err
}

// readWALRecord reads one WAL record from the reader.
// Returns (nil, nil) on clean EOF. Returns error on data corruption.
// A CRC mismatch indicates a truncated trailing record (expected on crash).
func readWALRecord(r io.Reader) (*WALMessage, error) {
	var magic uint32
	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
		if err == io.EOF {
			return nil, nil // Clean EOF
		}
		return nil, fmt.Errorf("read record magic: %w", err)
	}

	if magic != walRecordMagic {
		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
	}

	var payloadLen uint32
	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
		return nil, fmt.Errorf("read payload length: %w", err)
	}

	if payloadLen > 1<<20 { // 1 MB sanity limit
		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
	}

	payload := make([]byte, payloadLen)
	if _, err := io.ReadFull(r, payload); err != nil {
		return nil, fmt.Errorf("read payload: %w", err)
	}

	var storedCRC uint32
	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
		return nil, fmt.Errorf("read CRC: %w", err)
	}

	if crc32.ChecksumIEEE(payload) != storedCRC {
		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
	}

	return parseWALPayload(payload)
}

// parseWALPayload decodes a binary payload into a WALMessage.
func parseWALPayload(payload []byte) (*WALMessage, error) {
	if len(payload) < 8+2+1+4 {
		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
	}

	offset := 0

	// Timestamp (8 bytes)
	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
	offset += 8

	// Metric name (2-byte length + bytes)
	if offset+2 > len(payload) {
		return nil, fmt.Errorf("metric name length overflows payload")
	}
	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
	offset += 2

	if offset+mLen > len(payload) {
		return nil, fmt.Errorf("metric name overflows payload")
	}
	metricName := string(payload[offset : offset+mLen])
	offset += mLen

	// Selector count (1 byte)
	if offset >= len(payload) {
		return nil, fmt.Errorf("selector count overflows payload")
	}
	selCount := int(payload[offset])
	offset++

	selectors := make([]string, selCount)
	for i := range selCount {
		if offset >= len(payload) {
			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
		}
		sLen := int(payload[offset])
		offset++

		if offset+sLen > len(payload) {
			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
		}
		selectors[i] = string(payload[offset : offset+sLen])
		offset += sLen
	}

	// Value (4 bytes, float32 bits)
	if offset+4 > len(payload) {
		return nil, fmt.Errorf("value overflows payload")
	}
	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
	value := schema.Float(math.Float32frombits(bits))

	return &WALMessage{
		MetricName: metricName,
		Timestamp:  ts,
		Selector:   selectors,
		Value:      value,
	}, nil
}

// loadWALFile reads a WAL file and replays all valid records into the Level tree.
// l is the host-level node. Corrupt or partial trailing records are silently skipped
// (expected on crash). Records older than 'from' are skipped.
func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	// Verify file header magic.
	var fileMagic uint32
	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
		if err == io.EOF {
			return nil // Empty file, no data
		}
		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
	}

	if fileMagic != walFileMagic {
		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
	}

	// Cache level lookups to avoid repeated tree traversal.
	lvlCache := make(map[string]*Level)

	for {
		msg, err := readWALRecord(br)
		if err != nil {
			// Truncated trailing record is expected after a crash; stop replaying.
			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
			break
		}
		if msg == nil {
			break // Clean EOF
		}

		if msg.Timestamp < from {
			continue // Older than retention window
		}

		minfo, ok := m.Metrics[msg.MetricName]
		if !ok {
			continue // Unknown metric (config may have changed)
		}

		// Cache key is the null-separated selector path.
		cacheKey := joinSelector(msg.Selector)
		lvl, ok := lvlCache[cacheKey]
		if !ok {
			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
			lvlCache[cacheKey] = lvl
		}

		// Write directly to the buffer, same as WriteToLevel but without the
		// global level lookup (we already have the right level).
		lvl.lock.Lock()
		b := lvl.metrics[minfo.offset]
		if b == nil {
			b = newBuffer(msg.Timestamp, minfo.Frequency)
			lvl.metrics[minfo.offset] = b
		}
		nb, writeErr := b.write(msg.Timestamp, msg.Value)
		if writeErr == nil && b != nb {
			lvl.metrics[minfo.offset] = nb
		}
		// Ignore write errors for timestamps before buffer start (can happen when
		// replaying WAL entries that predate a loaded snapshot's start time).
		lvl.lock.Unlock()
	}

	return nil
}

// joinSelector builds a cache key from a selector slice using null bytes as separators.
func joinSelector(sel []string) string {
	if len(sel) == 0 {
		return ""
	}
	var result strings.Builder
	result.WriteString(sel[0])
	for i := 1; i < len(sel); i++ {
		result.WriteString("\x00" + sel[i])
	}
	return result.String()
}

// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
// Returns the number of files written, the list of host directories that were
// successfully checkpointed (for WAL rotation), and any errors.
func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
	// Collect all cluster/host pairs.
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		hostDir  string
		selector []string
	}

	totalWork := len(levels)
	cclog.Infof("[METRICSTORE]> Starting binary checkpoint for %d hosts with %d workers", totalWork, Keys.NumWorkers)

	n, errs, completed := int32(0), int32(0), int32(0)
	var successDirs []string
	var successMu sync.Mutex

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)

	// Progress logging goroutine.
	stopProgress := make(chan struct{})
	go func() {
		ticker := time.NewTicker(10 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				cclog.Infof("[METRICSTORE]> Checkpoint progress: %d/%d hosts (%d written, %d errors)",
					atomic.LoadInt32(&completed), totalWork, atomic.LoadInt32(&n), atomic.LoadInt32(&errs))
			case <-stopProgress:
				return
			}
		}
	}()

	for range Keys.NumWorkers {
		go func() {
			defer wg.Done()
			for wi := range work {
				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
				if err != nil {
					if err == ErrNoNewArchiveData {
						atomic.AddInt32(&completed, 1)
						continue
					}
					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
					successMu.Lock()
					successDirs = append(successDirs, wi.hostDir)
					successMu.Unlock()
				}
				atomic.AddInt32(&completed, 1)
			}
		}()
	}

	for i := range levels {
		hostDir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			hostDir:  hostDir,
			selector: selectors[i],
		}
	}
	close(work)
	wg.Wait()
	close(stopProgress)

	if errs > 0 {
		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
	}
	return int(n), successDirs, nil
}

// toCheckpointBinary writes a binary snapshot file for a single host-level node.
// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}
	if cf == nil {
		return ErrNoNewArchiveData
	}

	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
		return fmt.Errorf("mkdir %s: %w", dir, err)
	}

	// Write to a temp file first, then rename (atomic on POSIX).
	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))

	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
	}

	bw := bufio.NewWriter(f)
	if err := writeBinarySnapshotFile(bw, cf); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return fmt.Errorf("write binary snapshot: %w", err)
	}
	if err := bw.Flush(); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return err
	}
	f.Close()

	return os.Rename(tmpPath, finalPath)
}

// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
		return err
	}
	return writeBinaryLevel(w, cf)
}

// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
		return err
	}

	for name, metric := range cf.Metrics {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
			return err
		}
		for _, v := range metric.Data {
			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
				return err
			}
		}
	}

	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
		return err
	}

	for name, child := range cf.Children {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := writeBinaryLevel(w, child); err != nil {
			return err
		}
	}

	return nil
}

// writeString16 writes a 2-byte length-prefixed string to w.
func writeString16(w io.Writer, s string) error {
	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
		return err
	}
	_, err := io.WriteString(w, s)
	return err
}

// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
// The retention check (from) is applied to the file's 'to' timestamp.
func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	var magic uint32
	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
	}
	if magic != snapFileMagic {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
	}

	var fileFrom, fileTo int64
	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
	}
	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
	}

	if fileTo != 0 && fileTo < from {
		return nil // File is older than retention window, skip it
	}

	cf, err := readBinaryLevel(br)
	if err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
	}
	cf.From = fileFrom
	cf.To = fileTo

	return l.loadFile(cf, m)
}

// readBinaryLevel recursively reads a level from the binary snapshot format.
func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
	cf := &CheckpointFile{
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	var numMetrics uint32
	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
		return nil, fmt.Errorf("read num_metrics: %w", err)
	}

	for range numMetrics {
		name, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read metric name: %w", err)
		}

		var freq, start int64
		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
		}
		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
			return nil, fmt.Errorf("read start for %s: %w", name, err)
		}

		var numValues uint32
		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
		}

		data := make([]schema.Float, numValues)
		for i := range numValues {
			var bits uint32
			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
			}
			data[i] = schema.Float(math.Float32frombits(bits))
		}

		cf.Metrics[name] = &CheckpointMetrics{
			Frequency: freq,
			Start:     start,
			Data:      data,
		}
	}

	var numChildren uint32
	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
		return nil, fmt.Errorf("read num_children: %w", err)
	}

	for range numChildren {
		childName, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read child name: %w", err)
		}

		child, err := readBinaryLevel(r)
		if err != nil {
			return nil, fmt.Errorf("read child %s: %w", childName, err)
		}
		cf.Children[childName] = child
	}

	return cf, nil
}

// readString16 reads a 2-byte length-prefixed string from r.
func readString16(r io.Reader) (string, error) {
	var sLen uint16
	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
		return "", err
	}
	buf := make([]byte, sLen)
	if _, err := io.ReadFull(r, buf); err != nil {
		return "", err
	}
	return string(buf), nil
}


Now let me read the remaining files I haven't seen yet.


Let me read the remaining files.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides buffer.go: Time-series data buffer implementation.
     7→//
     8→// # Buffer Architecture
     9→//
    10→// Each metric at each hierarchical level (cluster/host/cpu/etc.) uses a linked-list
    11→// chain of fixed-size buffers to store time-series data. This design:
    12→//
    13→//   - Avoids reallocation/copying when growing (new links added instead)
    14→//   - Enables efficient pooling (buffers returned to sync.Pool)
    15→//   - Supports traversal back in time (via prev pointers)
    16→//   - Maintains temporal ordering (newer data in later buffers)
    17→//
    18→// # Buffer Chain Example
    19→//
    20→//	[oldest buffer] <- prev -- [older] <- prev -- [newest buffer (head)]
    21→//	  start=1000               start=1512           start=2024
    22→//	  data=[v0...v511]         data=[v0...v511]     data=[v0...v42]
    23→//
    24→// When the head buffer reaches capacity (BufferCap = 512), a new buffer becomes
    25→// the new head and the old head is linked via prev.
    26→//
    27→// # Pooling Strategy
    28→//
    29→// sync.Pool reduces GC pressure for the common case (BufferCap-sized allocations).
    30→// Non-standard capacity buffers are not pooled (e.g., from checkpoint deserialization).
    31→//
    32→// # Time Alignment
    33→//
    34→// Timestamps are aligned to measurement frequency intervals:
    35→//
    36→//	index = (timestamp - buffer.start) / buffer.frequency
    37→//	actualTime = buffer.start + (frequency / 2) + (index * frequency)
    38→//
    39→// Missing data points are represented as NaN values. The read() function performs
    40→// linear interpolation where possible.
    41→package metricstore
    42→
    43→import (
    44→	"errors"
    45→	"sync"
    46→	"time"
    47→
    48→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    49→)
    50→
    51→// BufferCap is the default buffer capacity.
    52→// buffer.data will only ever grow up to its capacity and a new link
    53→// in the buffer chain will be created if needed so that no copying
    54→// of data or reallocation needs to happen on writes.
    55→const BufferCap int = DefaultBufferCapacity
    56→
    57→// BufferPool is the global instance.
    58→// It is initialized immediately when the package loads.
    59→var bufferPool = NewPersistentBufferPool()
    60→
    61→type PersistentBufferPool struct {
    62→	pool []*buffer
    63→	mu   sync.Mutex
    64→}
    65→
    66→// NewPersistentBufferPool creates a dynamic pool for buffers.
    67→func NewPersistentBufferPool() *PersistentBufferPool {
    68→	return &PersistentBufferPool{
    69→		pool: make([]*buffer, 0),
    70→	}
    71→}
    72→
    73→func (p *PersistentBufferPool) Get() *buffer {
    74→	p.mu.Lock()
    75→	defer p.mu.Unlock()
    76→
    77→	n := len(p.pool)
    78→	if n == 0 {
    79→		// Pool is empty, allocate a new one
    80→		return &buffer{
    81→			data: make([]schema.Float, 0, BufferCap),
    82→		}
    83→	}
    84→
    85→	// Reuse existing buffer from the pool
    86→	b := p.pool[n-1]
    87→	p.pool[n-1] = nil // Avoid memory leak
    88→	p.pool = p.pool[:n-1]
    89→	return b
    90→}
    91→
    92→// Put returns b to the pool. The caller must set b.lastUsed = time.Now().Unix()
    93→// before calling Put so that Clean() can evict idle entries correctly.
    94→func (p *PersistentBufferPool) Put(b *buffer) {
    95→	// Reset the buffer before putting it back
    96→	b.data = b.data[:0]
    97→
    98→	p.mu.Lock()
    99→	defer p.mu.Unlock()
   100→
   101→	p.pool = append(p.pool, b)
   102→}
   103→
   104→// GetSize returns the exact number of buffers currently sitting in the pool.
   105→func (p *PersistentBufferPool) GetSize() int {
   106→	p.mu.Lock()
   107→	defer p.mu.Unlock()
   108→	return len(p.pool)
   109→}
   110→
   111→// Clear drains all buffers currently in the pool, allowing the GC to collect them.
   112→func (p *PersistentBufferPool) Clear() {
   113→	p.mu.Lock()
   114→	defer p.mu.Unlock()
   115→	for i := range p.pool {
   116→		p.pool[i] = nil
   117→	}
   118→	p.pool = p.pool[:0]
   119→}
   120→
   121→// Clean removes buffers from the pool that haven't been used in the given duration.
   122→// It uses a simple LRU approach based on the lastUsed timestamp.
   123→func (p *PersistentBufferPool) Clean(threshold int64) {
   124→	p.mu.Lock()
   125→	defer p.mu.Unlock()
   126→
   127→	// Filter in place, retaining only buffers returned to the pool recently enough.
   128→	active := p.pool[:0]
   129→	for _, b := range p.pool {
   130→		if b.lastUsed >= threshold {
   131→			active = append(active, b)
   132→		}
   133→	}
   134→
   135→	// Nullify the rest to prevent memory leaks
   136→	for i := len(active); i < len(p.pool); i++ {
   137→		p.pool[i] = nil
   138→	}
   139→
   140→	p.pool = active
   141→}
   142→
   143→var (
   144→	// ErrNoData indicates no time-series data exists for the requested metric/level.
   145→	ErrNoData error = errors.New("[METRICSTORE]> no data for this metric/level")
   146→
   147→	// ErrDataDoesNotAlign indicates that aggregated data from child scopes
   148→	// does not align with the parent scope's expected timestamps/intervals.
   149→	ErrDataDoesNotAlignMissingFront    error = errors.New("[METRICSTORE]> data from lower granularities does not align (missing data prior to start of the buffers)")
   150→	ErrDataDoesNotAlignMissingBack     error = errors.New("[METRICSTORE]> data from lower granularities does not align (missing data after the end of the buffers)")
   151→	ErrDataDoesNotAlignDataLenMismatch error = errors.New("[METRICSTORE]> data from lower granularities does not align (collected data length is different than expected data length)")
   152→)
   153→
   154→// buffer stores time-series data for a single metric at a specific hierarchical level.
   155→//
   156→// Buffers form doubly-linked chains ordered by time. When capacity is reached,
   157→// a new buffer becomes the head and the old head is linked via prev/next.
   158→//
   159→// Fields:
   160→//   - prev:      Link to older buffer in the chain (nil if this is oldest)
   161→//   - next:      Link to newer buffer in the chain (nil if this is newest/head)
   162→//   - data:      Time-series values (schema.Float supports NaN for missing data)
   163→//   - frequency: Measurement interval in seconds
   164→//   - start:     Start timestamp (adjusted by -frequency/2 for alignment)
   165→//   - archived:  True if data has been persisted to disk archive
   166→//   - closed:    True if buffer is no longer accepting writes
   167→//
   168→// Index calculation: index = (timestamp - start) / frequency
   169→// Actual data timestamp: start + (frequency / 2) + (index * frequency)
   170→type buffer struct {
   171→	prev      *buffer
   172→	next      *buffer
   173→	data      []schema.Float
   174→	frequency int64
   175→	start     int64
   176→	archived  bool
   177→	closed    bool
   178→	lastUsed  int64
   179→}
   180→
   181→func newBuffer(ts, freq int64) *buffer {
   182→	b := bufferPool.Get()
   183→	b.frequency = freq
   184→	b.start = ts - (freq / 2)
   185→	b.prev = nil
   186→	b.next = nil
   187→	b.archived = false
   188→	b.closed = false
   189→	b.data = b.data[:0]
   190→	return b
   191→}
   192→
   193→// write appends a timestamped value to the buffer chain.
   194→//
   195→// Returns the head buffer (which may be newly created if capacity was reached).
   196→// Timestamps older than the buffer's start are rejected. If the calculated index
   197→// exceeds capacity, a new buffer is allocated and linked as the new head.
   198→//
   199→// Missing timestamps are automatically filled with NaN values to maintain alignment.
   200→// Overwrites are allowed if the index is already within the existing data slice.
   201→//
   202→// Parameters:
   203→//   - ts:    Unix timestamp in seconds
   204→//   - value: Metric value (can be schema.NaN for missing data)
   205→//
   206→// Returns:
   207→//   - *buffer: The new head buffer (same as b if no new buffer created)
   208→//   - error:   Non-nil if timestamp is before buffer start
   209→func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
   210→	if ts < b.start {
   211→		return nil, errors.New("[METRICSTORE]> cannot write value to buffer from past")
   212→	}
   213→
   214→	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
   215→	idx := int((ts - b.start) / b.frequency)
   216→	if idx >= cap(b.data) {
   217→		newbuf := newBuffer(ts, b.frequency)
   218→		newbuf.prev = b
   219→		b.next = newbuf
   220→		b = newbuf
   221→		idx = 0
   222→	}
   223→
   224→	// Overwriting value or writing value from past
   225→	if idx < len(b.data) {
   226→		b.data[idx] = value
   227→		return b, nil
   228→	}
   229→
   230→	// Fill up unwritten slots with NaN
   231→	for i := len(b.data); i < idx; i++ {
   232→		b.data = append(b.data, schema.NaN)
   233→	}
   234→
   235→	b.data = append(b.data, value)
   236→	return b, nil
   237→}
   238→
   239→func (b *buffer) end() int64 {
   240→	return b.firstWrite() + int64(len(b.data))*b.frequency
   241→}
   242→
   243→func (b *buffer) firstWrite() int64 {
   244→	return b.start + (b.frequency / 2)
   245→}
   246→
   247→// read retrieves time-series data from the buffer chain for the specified time range.
   248→//
   249→// Traverses the buffer chain backwards (via prev links) if 'from' precedes the current
   250→// buffer's start. Missing data points are represented as NaN. Values are accumulated
   251→// into the provided 'data' slice (using +=, so caller must zero-initialize if needed).
   252→//
   253→// The function adjusts the actual time range returned if data is unavailable at the
   254→// boundaries (returned via adjusted from/to timestamps).
   255→//
   256→// Parameters:
   257→//   - from: Start timestamp (Unix seconds)
   258→//   - to:   End timestamp (Unix seconds, exclusive)
   259→//   - data: Pre-allocated slice to accumulate results (must be large enough)
   260→//
   261→// Returns:
   262→//   - []schema.Float: Slice of data (may be shorter than input 'data' slice)
   263→//   - int64:          Actual start timestamp with available data
   264→//   - int64:          Actual end timestamp (exclusive)
   265→//   - error:          Non-nil on failure
   266→//
   267→// Panics if 'data' slice is too small to hold all values in [from, to).
   268→func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) {
   269→	// Walk back to the buffer that covers 'from', adjusting if we hit the oldest.
   270→	for from < b.firstWrite() {
   271→		if b.prev == nil {
   272→			from = b.firstWrite()
   273→			break
   274→		}
   275→		b = b.prev
   276→	}
   277→
   278→	i := 0
   279→	t := from
   280→	for ; t < to; t += b.frequency {
   281→		idx := int((t - b.start) / b.frequency)
   282→		if idx >= cap(b.data) {
   283→			if b.next == nil {
   284→				break
   285→			}
   286→			b = b.next
   287→			// Recalculate idx in the new buffer; a gap between buffers may exist.
   288→			idx = int((t - b.start) / b.frequency)
   289→		}
   290→
   291→		if idx >= len(b.data) {
   292→			if b.next == nil || to <= b.next.start {
   293→				break
   294→			}
   295→			data[i] += schema.NaN // NaN + anything = NaN; propagates missing data
   296→		} else if t < b.start {
   297→			data[i] += schema.NaN // gap before this buffer's first write
   298→		} else {
   299→			data[i] += b.data[idx]
   300→		}
   301→		i++
   302→	}
   303→
   304→	return data[:i], from, t, nil
   305→}
   306→
   307→// free removes buffers older than the specified timestamp from the chain.
   308→//
   309→// Recursively traverses backwards (via prev) and unlinks buffers whose end time
   310→// is before the retention threshold. Freed buffers are returned to the pool if
   311→// they have the standard capacity (BufferCap).
   312→//
   313→// Parameters:
   314→//   - t: Retention threshold timestamp (Unix seconds)
   315→//
   316→// Returns:
   317→//   - delme: True if the current buffer itself should be deleted by caller
   318→//   - n:     Number of buffers freed in this subtree
   319→func (b *buffer) free(t int64) (delme bool, n int) {
   320→	if b.prev != nil {
   321→		delme, m := b.prev.free(t)
   322→		n += m
   323→		if delme {
   324→			b.prev.next = nil
   325→			if cap(b.prev.data) != BufferCap {
   326→				b.prev.data = make([]schema.Float, 0, BufferCap)
   327→			}
   328→			b.prev.lastUsed = time.Now().Unix()
   329→			bufferPool.Put(b.prev)
   330→			b.prev = nil
   331→		}
   332→	}
   333→
   334→	end := b.end()
   335→	if end < t {
   336→		return true, n + 1
   337→	}
   338→
   339→	return false, n
   340→}
   341→
   342→// forceFreeOldest recursively finds the end of the linked list (the oldest buffer)
   343→// and removes it.
   344→// Returns:
   345→//
   346→//	delme: true if 'b' itself is the oldest and should be removed by the caller
   347→//	n:     the number of buffers freed (will be 1 or 0)
   348→func (b *buffer) forceFreeOldest() (delme bool, n int) {
   349→	// If there is a previous buffer, recurse down to find the oldest
   350→	if b.prev != nil {
   351→		delPrev, freed := b.prev.forceFreeOldest()
   352→
   353→		// If the previous buffer signals it should be deleted:
   354→		if delPrev {
   355→			b.prev.next = nil
   356→			if cap(b.prev.data) != BufferCap {
   357→				b.prev.data = make([]schema.Float, 0, BufferCap)
   358→			}
   359→			b.prev.lastUsed = time.Now().Unix()
   360→			bufferPool.Put(b.prev)
   361→			b.prev = nil
   362→		}
   363→		return false, freed
   364→	}
   365→
   366→	// If b.prev is nil, THIS buffer is the oldest.
   367→	// We return true so the parent (or the Level loop) knows to delete reference to 'b'.
   368→	return true, 1
   369→}
   370→
   371→// iterFromTo invokes callback on every buffer in the chain that overlaps [from, to].
   372→//
   373→// Traverses backwards (via prev) first, then processes current buffer if it overlaps
   374→// the time range. Used for checkpoint/archive operations that need to serialize buffers
   375→// within a specific time window.
   376→//
   377→// Parameters:
   378→//   - from:     Start timestamp (Unix seconds, inclusive)
   379→//   - to:       End timestamp (Unix seconds, inclusive)
   380→//   - callback: Function to invoke on each overlapping buffer
   381→//
   382→// Returns:
   383→//   - error: First error returned by callback, or nil if all succeeded
   384→func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error {
   385→	if b == nil {
   386→		return nil
   387→	}
   388→
   389→	// Collect overlapping buffers walking backwards (newest → oldest).
   390→	var matching []*buffer
   391→	for cur := b; cur != nil; cur = cur.prev {
   392→		if from <= cur.end() && cur.start <= to {
   393→			matching = append(matching, cur)
   394→		}
   395→	}
   396→
   397→	// Invoke callback in chronological order (oldest → newest).
   398→	for i := len(matching) - 1; i >= 0; i-- {
   399→		if err := callback(matching[i]); err != nil {
   400→			return err
   401→		}
   402→	}
   403→	return nil
   404→}
   405→
   406→func (b *buffer) count() int64 {
   407→	var res int64
   408→	for ; b != nil; b = b.prev {
   409→		res += int64(len(b.data))
   410→	}
   411→	return res
   412→}
   413→
   414→// bufferCount returns the number of buffer nodes in the linked list.
   415→func (b *buffer) bufferCount() int64 {
   416→	var n int64
   417→	for ; b != nil; b = b.prev {
   418→		n++
   419→	}
   420→	return n
   421→}
   422→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides buffer.go: Time-series data buffer implementation.
//
// # Buffer Architecture
//
// Each metric at each hierarchical level (cluster/host/cpu/etc.) uses a linked-list
// chain of fixed-size buffers to store time-series data. This design:
//
//   - Avoids reallocation/copying when growing (new links added instead)
//   - Enables efficient pooling (buffers returned to sync.Pool)
//   - Supports traversal back in time (via prev pointers)
//   - Maintains temporal ordering (newer data in later buffers)
//
// # Buffer Chain Example
//
//	[oldest buffer] <- prev -- [older] <- prev -- [newest buffer (head)]
//	  start=1000               start=1512           start=2024
//	  data=[v0...v511]         data=[v0...v511]     data=[v0...v42]
//
// When the head buffer reaches capacity (BufferCap = 512), a new buffer becomes
// the new head and the old head is linked via prev.
//
// # Pooling Strategy
//
// sync.Pool reduces GC pressure for the common case (BufferCap-sized allocations).
// Non-standard capacity buffers are not pooled (e.g., from checkpoint deserialization).
//
// # Time Alignment
//
// Timestamps are aligned to measurement frequency intervals:
//
//	index = (timestamp - buffer.start) / buffer.frequency
//	actualTime = buffer.start + (frequency / 2) + (index * frequency)
//
// Missing data points are represented as NaN values. The read() function performs
// linear interpolation where possible.
package metricstore

import (
	"errors"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// BufferCap is the default buffer capacity.
// buffer.data will only ever grow up to its capacity and a new link
// in the buffer chain will be created if needed so that no copying
// of data or reallocation needs to happen on writes.
const BufferCap int = DefaultBufferCapacity

// BufferPool is the global instance.
// It is initialized immediately when the package loads.
var bufferPool = NewPersistentBufferPool()

type PersistentBufferPool struct {
	pool []*buffer
	mu   sync.Mutex
}

// NewPersistentBufferPool creates a dynamic pool for buffers.
func NewPersistentBufferPool() *PersistentBufferPool {
	return &PersistentBufferPool{
		pool: make([]*buffer, 0),
	}
}

func (p *PersistentBufferPool) Get() *buffer {
	p.mu.Lock()
	defer p.mu.Unlock()

	n := len(p.pool)
	if n == 0 {
		// Pool is empty, allocate a new one
		return &buffer{
			data: make([]schema.Float, 0, BufferCap),
		}
	}

	// Reuse existing buffer from the pool
	b := p.pool[n-1]
	p.pool[n-1] = nil // Avoid memory leak
	p.pool = p.pool[:n-1]
	return b
}

// Put returns b to the pool. The caller must set b.lastUsed = time.Now().Unix()
// before calling Put so that Clean() can evict idle entries correctly.
func (p *PersistentBufferPool) Put(b *buffer) {
	// Reset the buffer before putting it back
	b.data = b.data[:0]

	p.mu.Lock()
	defer p.mu.Unlock()

	p.pool = append(p.pool, b)
}

// GetSize returns the exact number of buffers currently sitting in the pool.
func (p *PersistentBufferPool) GetSize() int {
	p.mu.Lock()
	defer p.mu.Unlock()
	return len(p.pool)
}

// Clear drains all buffers currently in the pool, allowing the GC to collect them.
func (p *PersistentBufferPool) Clear() {
	p.mu.Lock()
	defer p.mu.Unlock()
	for i := range p.pool {
		p.pool[i] = nil
	}
	p.pool = p.pool[:0]
}

// Clean removes buffers from the pool that haven't been used in the given duration.
// It uses a simple LRU approach based on the lastUsed timestamp.
func (p *PersistentBufferPool) Clean(threshold int64) {
	p.mu.Lock()
	defer p.mu.Unlock()

	// Filter in place, retaining only buffers returned to the pool recently enough.
	active := p.pool[:0]
	for _, b := range p.pool {
		if b.lastUsed >= threshold {
			active = append(active, b)
		}
	}

	// Nullify the rest to prevent memory leaks
	for i := len(active); i < len(p.pool); i++ {
		p.pool[i] = nil
	}

	p.pool = active
}

var (
	// ErrNoData indicates no time-series data exists for the requested metric/level.
	ErrNoData error = errors.New("[METRICSTORE]> no data for this metric/level")

	// ErrDataDoesNotAlign indicates that aggregated data from child scopes
	// does not align with the parent scope's expected timestamps/intervals.
	ErrDataDoesNotAlignMissingFront    error = errors.New("[METRICSTORE]> data from lower granularities does not align (missing data prior to start of the buffers)")
	ErrDataDoesNotAlignMissingBack     error = errors.New("[METRICSTORE]> data from lower granularities does not align (missing data after the end of the buffers)")
	ErrDataDoesNotAlignDataLenMismatch error = errors.New("[METRICSTORE]> data from lower granularities does not align (collected data length is different than expected data length)")
)

// buffer stores time-series data for a single metric at a specific hierarchical level.
//
// Buffers form doubly-linked chains ordered by time. When capacity is reached,
// a new buffer becomes the head and the old head is linked via prev/next.
//
// Fields:
//   - prev:      Link to older buffer in the chain (nil if this is oldest)
//   - next:      Link to newer buffer in the chain (nil if this is newest/head)
//   - data:      Time-series values (schema.Float supports NaN for missing data)
//   - frequency: Measurement interval in seconds
//   - start:     Start timestamp (adjusted by -frequency/2 for alignment)
//   - archived:  True if data has been persisted to disk archive
//   - closed:    True if buffer is no longer accepting writes
//
// Index calculation: index = (timestamp - start) / frequency
// Actual data timestamp: start + (frequency / 2) + (index * frequency)
type buffer struct {
	prev      *buffer
	next      *buffer
	data      []schema.Float
	frequency int64
	start     int64
	archived  bool
	closed    bool
	lastUsed  int64
}

func newBuffer(ts, freq int64) *buffer {
	b := bufferPool.Get()
	b.frequency = freq
	b.start = ts - (freq / 2)
	b.prev = nil
	b.next = nil
	b.archived = false
	b.closed = false
	b.data = b.data[:0]
	return b
}

// write appends a timestamped value to the buffer chain.
//
// Returns the head buffer (which may be newly created if capacity was reached).
// Timestamps older than the buffer's start are rejected. If the calculated index
// exceeds capacity, a new buffer is allocated and linked as the new head.
//
// Missing timestamps are automatically filled with NaN values to maintain alignment.
// Overwrites are allowed if the index is already within the existing data slice.
//
// Parameters:
//   - ts:    Unix timestamp in seconds
//   - value: Metric value (can be schema.NaN for missing data)
//
// Returns:
//   - *buffer: The new head buffer (same as b if no new buffer created)
//   - error:   Non-nil if timestamp is before buffer start
func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
	if ts < b.start {
		return nil, errors.New("[METRICSTORE]> cannot write value to buffer from past")
	}

	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
	idx := int((ts - b.start) / b.frequency)
	if idx >= cap(b.data) {
		newbuf := newBuffer(ts, b.frequency)
		newbuf.prev = b
		b.next = newbuf
		b = newbuf
		idx = 0
	}

	// Overwriting value or writing value from past
	if idx < len(b.data) {
		b.data[idx] = value
		return b, nil
	}

	// Fill up unwritten slots with NaN
	for i := len(b.data); i < idx; i++ {
		b.data = append(b.data, schema.NaN)
	}

	b.data = append(b.data, value)
	return b, nil
}

func (b *buffer) end() int64 {
	return b.firstWrite() + int64(len(b.data))*b.frequency
}

func (b *buffer) firstWrite() int64 {
	return b.start + (b.frequency / 2)
}

// read retrieves time-series data from the buffer chain for the specified time range.
//
// Traverses the buffer chain backwards (via prev links) if 'from' precedes the current
// buffer's start. Missing data points are represented as NaN. Values are accumulated
// into the provided 'data' slice (using +=, so caller must zero-initialize if needed).
//
// The function adjusts the actual time range returned if data is unavailable at the
// boundaries (returned via adjusted from/to timestamps).
//
// Parameters:
//   - from: Start timestamp (Unix seconds)
//   - to:   End timestamp (Unix seconds, exclusive)
//   - data: Pre-allocated slice to accumulate results (must be large enough)
//
// Returns:
//   - []schema.Float: Slice of data (may be shorter than input 'data' slice)
//   - int64:          Actual start timestamp with available data
//   - int64:          Actual end timestamp (exclusive)
//   - error:          Non-nil on failure
//
// Panics if 'data' slice is too small to hold all values in [from, to).
func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) {
	// Walk back to the buffer that covers 'from', adjusting if we hit the oldest.
	for from < b.firstWrite() {
		if b.prev == nil {
			from = b.firstWrite()
			break
		}
		b = b.prev
	}

	i := 0
	t := from
	for ; t < to; t += b.frequency {
		idx := int((t - b.start) / b.frequency)
		if idx >= cap(b.data) {
			if b.next == nil {
				break
			}
			b = b.next
			// Recalculate idx in the new buffer; a gap between buffers may exist.
			idx = int((t - b.start) / b.frequency)
		}

		if idx >= len(b.data) {
			if b.next == nil || to <= b.next.start {
				break
			}
			data[i] += schema.NaN // NaN + anything = NaN; propagates missing data
		} else if t < b.start {
			data[i] += schema.NaN // gap before this buffer's first write
		} else {
			data[i] += b.data[idx]
		}
		i++
	}

	return data[:i], from, t, nil
}

// free removes buffers older than the specified timestamp from the chain.
//
// Recursively traverses backwards (via prev) and unlinks buffers whose end time
// is before the retention threshold. Freed buffers are returned to the pool if
// they have the standard capacity (BufferCap).
//
// Parameters:
//   - t: Retention threshold timestamp (Unix seconds)
//
// Returns:
//   - delme: True if the current buffer itself should be deleted by caller
//   - n:     Number of buffers freed in this subtree
func (b *buffer) free(t int64) (delme bool, n int) {
	if b.prev != nil {
		delme, m := b.prev.free(t)
		n += m
		if delme {
			b.prev.next = nil
			if cap(b.prev.data) != BufferCap {
				b.prev.data = make([]schema.Float, 0, BufferCap)
			}
			b.prev.lastUsed = time.Now().Unix()
			bufferPool.Put(b.prev)
			b.prev = nil
		}
	}

	end := b.end()
	if end < t {
		return true, n + 1
	}

	return false, n
}

// forceFreeOldest recursively finds the end of the linked list (the oldest buffer)
// and removes it.
// Returns:
//
//	delme: true if 'b' itself is the oldest and should be removed by the caller
//	n:     the number of buffers freed (will be 1 or 0)
func (b *buffer) forceFreeOldest() (delme bool, n int) {
	// If there is a previous buffer, recurse down to find the oldest
	if b.prev != nil {
		delPrev, freed := b.prev.forceFreeOldest()

		// If the previous buffer signals it should be deleted:
		if delPrev {
			b.prev.next = nil
			if cap(b.prev.data) != BufferCap {
				b.prev.data = make([]schema.Float, 0, BufferCap)
			}
			b.prev.lastUsed = time.Now().Unix()
			bufferPool.Put(b.prev)
			b.prev = nil
		}
		return false, freed
	}

	// If b.prev is nil, THIS buffer is the oldest.
	// We return true so the parent (or the Level loop) knows to delete reference to 'b'.
	return true, 1
}

// iterFromTo invokes callback on every buffer in the chain that overlaps [from, to].
//
// Traverses backwards (via prev) first, then processes current buffer if it overlaps
// the time range. Used for checkpoint/archive operations that need to serialize buffers
// within a specific time window.
//
// Parameters:
//   - from:     Start timestamp (Unix seconds, inclusive)
//   - to:       End timestamp (Unix seconds, inclusive)
//   - callback: Function to invoke on each overlapping buffer
//
// Returns:
//   - error: First error returned by callback, or nil if all succeeded
func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error {
	if b == nil {
		return nil
	}

	// Collect overlapping buffers walking backwards (newest → oldest).
	var matching []*buffer
	for cur := b; cur != nil; cur = cur.prev {
		if from <= cur.end() && cur.start <= to {
			matching = append(matching, cur)
		}
	}

	// Invoke callback in chronological order (oldest → newest).
	for i := len(matching) - 1; i >= 0; i-- {
		if err := callback(matching[i]); err != nil {
			return err
		}
	}
	return nil
}

func (b *buffer) count() int64 {
	var res int64
	for ; b != nil; b = b.prev {
		res += int64(len(b.data))
	}
	return res
}

// bufferCount returns the number of buffer nodes in the linked list.
func (b *buffer) bufferCount() int64 {
	var n int64
	for ; b != nil; b = b.prev {
		n++
	}
	return n
}