Implement the following plan:

# Remaining Improvements for Metricstore

## IMPROVEMENT 1: WAL buffering — RECOMMEND
- `walFileState` (walCheckpoint.go:110) currently holds only `f *os.File`
- Each `writeWALRecord` call (line 307) does a raw `f.Write()` syscall
- Add `w *bufio.Writer` to `walFileState`, wrap `f` on creation (line 163)
- `writeWALRecord` writes to `bufio.Writer` instead of `*os.File`
- Flush in `processRotate` (line 178) before Close
- Flush in defer cleanup (line 126) before Close
- Flush in `drain()` (line 191) after processing remaining messages

## IMPROVEMENT 3: MemoryCap comment fix — RECOMMEND (trivial)
- `config.go:116` comment says "Max bytes" but code treats it as GB
- Fix comment to say "Max memory in GB"

## Files to modify
- `pkg/metricstore/walCheckpoint.go` — IMP 1: add `bufio.Writer` to `walFileState`, use in `writeWALRecord`, flush on rotate/close/drain
- `pkg/metricstore/config.go:116` — IMP 3: fix comment "Max bytes" → "Max memory in GB"

## NOT recommended now
- **IMP 4 (WAL size limit)**: High complexity, checkpoint-interval (already configurable) is the simpler lever
- **Bug 5 (periodic fsync)**: CRC handles corrupt trailing records; WAL is best-effort on top of snapshots

## Verification
```bash
go test ./pkg/metricstore/...
```

If you need specific details from before exiting plan mode (like exact code snippets, error messages, or content you generated), read the full transcript at: /Users/jan/.claude/projects/-Users-jan-prg-ClusterCockpit-cc-backend/f12048ec-dcf9-4d07-a9c1-0916c9882538.jsonl


Let me read the files I need to modify.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
     7→//
     8→// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
     9→// per host, eliminating the extra memory overhead of the AvroStore and providing
    10→// truly continuous (per-write) crash safety.
    11→//
    12→// # Architecture
    13→//
    14→//	Metric write (DecodeLine)
    15→//	      │
    16→//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
    17→//	      │
    18→//	      └─► WALMessages channel
    19→//	               │
    20→//	               ▼
    21→//	        WALStaging goroutine
    22→//	               │
    23→//	               ▼
    24→//	        checkpoints/cluster/host/current.wal  (append-only, binary)
    25→//
    26→//	Periodic checkpoint (Checkpointing goroutine):
    27→//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
    28→//	  2. Signal WALStaging to truncate current.wal per host
    29→//
    30→//	On restart (FromCheckpoint):
    31→//	  1. Load most recent <timestamp>.bin snapshot
    32→//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
    33→//
    34→// # WAL Record Format
    35→//
    36→//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
    37→//
    38→//	payload:
    39→//	  [8B timestamp int64]
    40→//	  [2B metric_name_len uint16][N metric name bytes]
    41→//	  [1B selector_count uint8]
    42→//	  per selector: [1B selector_len uint8][M selector bytes]
    43→//	  [4B value float32 bits]
    44→//
    45→// # Binary Snapshot Format
    46→//
    47→//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
    48→//	Level tree (recursive):
    49→//	  [4B num_metrics uint32]
    50→//	  per metric:
    51→//	    [2B name_len uint16][N name bytes]
    52→//	    [8B frequency int64][8B start int64]
    53→//	    [4B num_values uint32][num_values × 4B float32]
    54→//	  [4B num_children uint32]
    55→//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
    56→package metricstore
    57→
    58→import (
    59→	"bufio"
    60→	"context"
    61→	"encoding/binary"
    62→	"fmt"
    63→	"hash/crc32"
    64→	"io"
    65→	"math"
    66→	"os"
    67→	"path"
    68→	"strings"
    69→	"sync"
    70→	"sync/atomic"
    71→
    72→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    73→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    74→)
    75→
    76→// Magic numbers for binary formats.
    77→const (
    78→	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
    79→	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
    80→	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
    81→)
    82→
    83→// WALMessages is the channel for sending metric writes to the WAL staging goroutine.
    84→// Buffered to allow burst writes without blocking the metric ingestion path.
    85→var WALMessages = make(chan *WALMessage, 4096)
    86→
    87→// walRotateCh is used by the checkpoint goroutine to request WAL file rotation
    88→// (close, delete, reopen) after a binary snapshot has been written.
    89→var walRotateCh = make(chan walRotateReq, 256)
    90→
    91→// WALMessage represents a single metric write to be appended to the WAL.
    92→// Cluster and Node are NOT stored in the WAL record (inferred from file path).
    93→type WALMessage struct {
    94→	MetricName string
    95→	Cluster    string
    96→	Node       string
    97→	Selector   []string
    98→	Value      schema.Float
    99→	Timestamp  int64
   100→}
   101→
   102→// walRotateReq requests WAL file rotation for a specific host directory.
   103→// The done channel is closed by the WAL goroutine when rotation is complete.
   104→type walRotateReq struct {
   105→	hostDir string
   106→	done    chan struct{}
   107→}
   108→
   109→// walFileState holds an open WAL file handle for one host directory.
   110→type walFileState struct {
   111→	f *os.File
   112→}
   113→
   114→// WALStaging starts a background goroutine that receives WALMessage items
   115→// and appends binary WAL records to per-host current.wal files.
   116→// Also handles WAL rotation requests from the checkpoint goroutine.
   117→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   118→	wg.Go(func() {
   119→		if Keys.Checkpoints.FileFormat == "json" {
   120→			return
   121→		}
   122→
   123→		hostFiles := make(map[string]*walFileState)
   124→
   125→		defer func() {
   126→			for _, ws := range hostFiles {
   127→				if ws.f != nil {
   128→					ws.f.Close()
   129→				}
   130→			}
   131→		}()
   132→
   133→		getOrOpenWAL := func(hostDir string) *os.File {
   134→			ws, ok := hostFiles[hostDir]
   135→			if ok {
   136→				return ws.f
   137→			}
   138→
   139→			if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   140→				cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   141→				return nil
   142→			}
   143→
   144→			walPath := path.Join(hostDir, "current.wal")
   145→			f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   146→			if err != nil {
   147→				cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   148→				return nil
   149→			}
   150→
   151→			// Write file header magic if file is new (empty).
   152→			info, err := f.Stat()
   153→			if err == nil && info.Size() == 0 {
   154→				var hdr [4]byte
   155→				binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   156→				if _, err := f.Write(hdr[:]); err != nil {
   157→					cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   158→					f.Close()
   159→					return nil
   160→				}
   161→			}
   162→
   163→			hostFiles[hostDir] = &walFileState{f: f}
   164→			return f
   165→		}
   166→
   167→		processMsg := func(msg *WALMessage) {
   168→			hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   169→			f := getOrOpenWAL(hostDir)
   170→			if f == nil {
   171→				return
   172→			}
   173→			if err := writeWALRecord(f, msg); err != nil {
   174→				cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   175→			}
   176→		}
   177→
   178→		processRotate := func(req walRotateReq) {
   179→			ws, ok := hostFiles[req.hostDir]
   180→			if ok && ws.f != nil {
   181→				ws.f.Close()
   182→				walPath := path.Join(req.hostDir, "current.wal")
   183→				if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   184→					cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   185→				}
   186→				delete(hostFiles, req.hostDir)
   187→			}
   188→			close(req.done)
   189→		}
   190→
   191→		drain := func() {
   192→			for {
   193→				select {
   194→				case msg, ok := <-WALMessages:
   195→					if !ok {
   196→						return
   197→					}
   198→					processMsg(msg)
   199→				case req := <-walRotateCh:
   200→					processRotate(req)
   201→				default:
   202→					return
   203→				}
   204→			}
   205→		}
   206→
   207→		for {
   208→			select {
   209→			case <-ctx.Done():
   210→				drain()
   211→				return
   212→			case msg, ok := <-WALMessages:
   213→				if !ok {
   214→					return
   215→				}
   216→				processMsg(msg)
   217→			case req := <-walRotateCh:
   218→				processRotate(req)
   219→			}
   220→		}
   221→	})
   222→}
   223→
   224→// RotateWALFiles sends rotation requests for the given host directories
   225→// and blocks until all rotations complete.
   226→func RotateWALFiles(hostDirs []string) {
   227→	dones := make([]chan struct{}, len(hostDirs))
   228→	for i, dir := range hostDirs {
   229→		dones[i] = make(chan struct{})
   230→		walRotateCh <- walRotateReq{hostDir: dir, done: dones[i]}
   231→	}
   232→	for _, done := range dones {
   233→		<-done
   234→	}
   235→}
   236→
   237→// RotateWALFiles sends rotation requests for the given host directories
   238→// and blocks until all rotations complete.
   239→func RotateWALFilesAfterShutdown(hostDirs []string) {
   240→	for _, dir := range hostDirs {
   241→		walPath := path.Join(dir, "current.wal")
   242→		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   243→			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   244→		}
   245→	}
   246→}
   247→
   248→// buildWALPayload encodes a WALMessage into a binary payload (without magic/length/CRC).
   249→func buildWALPayload(msg *WALMessage) []byte {
   250→	size := 8 + 2 + len(msg.MetricName) + 1 + 4
   251→	for _, s := range msg.Selector {
   252→		size += 1 + len(s)
   253→	}
   254→
   255→	buf := make([]byte, 0, size)
   256→
   257→	// Timestamp (8 bytes, little-endian int64)
   258→	var ts [8]byte
   259→	binary.LittleEndian.PutUint64(ts[:], uint64(msg.Timestamp))
   260→	buf = append(buf, ts[:]...)
   261→
   262→	// Metric name (2-byte length prefix + bytes)
   263→	var mLen [2]byte
   264→	binary.LittleEndian.PutUint16(mLen[:], uint16(len(msg.MetricName)))
   265→	buf = append(buf, mLen[:]...)
   266→	buf = append(buf, msg.MetricName...)
   267→
   268→	// Selector count (1 byte)
   269→	buf = append(buf, byte(len(msg.Selector)))
   270→
   271→	// Selectors (1-byte length prefix + bytes each)
   272→	for _, sel := range msg.Selector {
   273→		buf = append(buf, byte(len(sel)))
   274→		buf = append(buf, sel...)
   275→	}
   276→
   277→	// Value (4 bytes, float32 bit representation)
   278→	var val [4]byte
   279→	binary.LittleEndian.PutUint32(val[:], math.Float32bits(float32(msg.Value)))
   280→	buf = append(buf, val[:]...)
   281→
   282→	return buf
   283→}
   284→
   285→// writeWALRecord appends a binary WAL record to the file.
   286→// Format: [4B magic][4B payload_len][payload][4B CRC32]
   287→func writeWALRecord(f *os.File, msg *WALMessage) error {
   288→	payload := buildWALPayload(msg)
   289→	crc := crc32.ChecksumIEEE(payload)
   290→
   291→	record := make([]byte, 0, 4+4+len(payload)+4)
   292→
   293→	var magic [4]byte
   294→	binary.LittleEndian.PutUint32(magic[:], walRecordMagic)
   295→	record = append(record, magic[:]...)
   296→
   297→	var pLen [4]byte
   298→	binary.LittleEndian.PutUint32(pLen[:], uint32(len(payload)))
   299→	record = append(record, pLen[:]...)
   300→
   301→	record = append(record, payload...)
   302→
   303→	var crcBytes [4]byte
   304→	binary.LittleEndian.PutUint32(crcBytes[:], crc)
   305→	record = append(record, crcBytes[:]...)
   306→
   307→	_, err := f.Write(record)
   308→	return err
   309→}
   310→
   311→// readWALRecord reads one WAL record from the reader.
   312→// Returns (nil, nil) on clean EOF. Returns error on data corruption.
   313→// A CRC mismatch indicates a truncated trailing record (expected on crash).
   314→func readWALRecord(r io.Reader) (*WALMessage, error) {
   315→	var magic uint32
   316→	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
   317→		if err == io.EOF {
   318→			return nil, nil // Clean EOF
   319→		}
   320→		return nil, fmt.Errorf("read record magic: %w", err)
   321→	}
   322→
   323→	if magic != walRecordMagic {
   324→		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
   325→	}
   326→
   327→	var payloadLen uint32
   328→	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
   329→		return nil, fmt.Errorf("read payload length: %w", err)
   330→	}
   331→
   332→	if payloadLen > 1<<20 { // 1 MB sanity limit
   333→		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
   334→	}
   335→
   336→	payload := make([]byte, payloadLen)
   337→	if _, err := io.ReadFull(r, payload); err != nil {
   338→		return nil, fmt.Errorf("read payload: %w", err)
   339→	}
   340→
   341→	var storedCRC uint32
   342→	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
   343→		return nil, fmt.Errorf("read CRC: %w", err)
   344→	}
   345→
   346→	if crc32.ChecksumIEEE(payload) != storedCRC {
   347→		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
   348→	}
   349→
   350→	return parseWALPayload(payload)
   351→}
   352→
   353→// parseWALPayload decodes a binary payload into a WALMessage.
   354→func parseWALPayload(payload []byte) (*WALMessage, error) {
   355→	if len(payload) < 8+2+1+4 {
   356→		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
   357→	}
   358→
   359→	offset := 0
   360→
   361→	// Timestamp (8 bytes)
   362→	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
   363→	offset += 8
   364→
   365→	// Metric name (2-byte length + bytes)
   366→	if offset+2 > len(payload) {
   367→		return nil, fmt.Errorf("metric name length overflows payload")
   368→	}
   369→	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
   370→	offset += 2
   371→
   372→	if offset+mLen > len(payload) {
   373→		return nil, fmt.Errorf("metric name overflows payload")
   374→	}
   375→	metricName := string(payload[offset : offset+mLen])
   376→	offset += mLen
   377→
   378→	// Selector count (1 byte)
   379→	if offset >= len(payload) {
   380→		return nil, fmt.Errorf("selector count overflows payload")
   381→	}
   382→	selCount := int(payload[offset])
   383→	offset++
   384→
   385→	selectors := make([]string, selCount)
   386→	for i := range selCount {
   387→		if offset >= len(payload) {
   388→			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
   389→		}
   390→		sLen := int(payload[offset])
   391→		offset++
   392→
   393→		if offset+sLen > len(payload) {
   394→			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
   395→		}
   396→		selectors[i] = string(payload[offset : offset+sLen])
   397→		offset += sLen
   398→	}
   399→
   400→	// Value (4 bytes, float32 bits)
   401→	if offset+4 > len(payload) {
   402→		return nil, fmt.Errorf("value overflows payload")
   403→	}
   404→	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
   405→	value := schema.Float(math.Float32frombits(bits))
   406→
   407→	return &WALMessage{
   408→		MetricName: metricName,
   409→		Timestamp:  ts,
   410→		Selector:   selectors,
   411→		Value:      value,
   412→	}, nil
   413→}
   414→
   415→// loadWALFile reads a WAL file and replays all valid records into the Level tree.
   416→// l is the host-level node. Corrupt or partial trailing records are silently skipped
   417→// (expected on crash). Records older than 'from' are skipped.
   418→func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
   419→	br := bufio.NewReader(f)
   420→
   421→	// Verify file header magic.
   422→	var fileMagic uint32
   423→	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
   424→		if err == io.EOF {
   425→			return nil // Empty file, no data
   426→		}
   427→		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
   428→	}
   429→
   430→	if fileMagic != walFileMagic {
   431→		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
   432→	}
   433→
   434→	// Cache level lookups to avoid repeated tree traversal.
   435→	lvlCache := make(map[string]*Level)
   436→
   437→	for {
   438→		msg, err := readWALRecord(br)
   439→		if err != nil {
   440→			// Truncated trailing record is expected after a crash; stop replaying.
   441→			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
   442→			break
   443→		}
   444→		if msg == nil {
   445→			break // Clean EOF
   446→		}
   447→
   448→		if msg.Timestamp < from {
   449→			continue // Older than retention window
   450→		}
   451→
   452→		minfo, ok := m.Metrics[msg.MetricName]
   453→		if !ok {
   454→			continue // Unknown metric (config may have changed)
   455→		}
   456→
   457→		// Cache key is the null-separated selector path.
   458→		cacheKey := joinSelector(msg.Selector)
   459→		lvl, ok := lvlCache[cacheKey]
   460→		if !ok {
   461→			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
   462→			lvlCache[cacheKey] = lvl
   463→		}
   464→
   465→		// Write directly to the buffer, same as WriteToLevel but without the
   466→		// global level lookup (we already have the right level).
   467→		lvl.lock.Lock()
   468→		b := lvl.metrics[minfo.offset]
   469→		if b == nil {
   470→			b = newBuffer(msg.Timestamp, minfo.Frequency)
   471→			lvl.metrics[minfo.offset] = b
   472→		}
   473→		nb, writeErr := b.write(msg.Timestamp, msg.Value)
   474→		if writeErr == nil && b != nb {
   475→			lvl.metrics[minfo.offset] = nb
   476→		}
   477→		// Ignore write errors for timestamps before buffer start (can happen when
   478→		// replaying WAL entries that predate a loaded snapshot's start time).
   479→		lvl.lock.Unlock()
   480→	}
   481→
   482→	return nil
   483→}
   484→
   485→// joinSelector builds a cache key from a selector slice using null bytes as separators.
   486→func joinSelector(sel []string) string {
   487→	if len(sel) == 0 {
   488→		return ""
   489→	}
   490→	var result strings.Builder
   491→	result.WriteString(sel[0])
   492→	for i := 1; i < len(sel); i++ {
   493→		result.WriteString("\x00" + sel[i])
   494→	}
   495→	return result.String()
   496→}
   497→
   498→// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
   499→// Returns the number of files written, the list of host directories that were
   500→// successfully checkpointed (for WAL rotation), and any errors.
   501→func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
   502→	// Collect all cluster/host pairs.
   503→	m.root.lock.RLock()
   504→	totalHosts := 0
   505→	for _, l1 := range m.root.children {
   506→		l1.lock.RLock()
   507→		totalHosts += len(l1.children)
   508→		l1.lock.RUnlock()
   509→	}
   510→	m.root.lock.RUnlock()
   511→
   512→	levels := make([]*Level, 0, totalHosts)
   513→	selectors := make([][]string, 0, totalHosts)
   514→
   515→	m.root.lock.RLock()
   516→	for sel1, l1 := range m.root.children {
   517→		l1.lock.RLock()
   518→		for sel2, l2 := range l1.children {
   519→			levels = append(levels, l2)
   520→			selectors = append(selectors, []string{sel1, sel2})
   521→		}
   522→		l1.lock.RUnlock()
   523→	}
   524→	m.root.lock.RUnlock()
   525→
   526→	type workItem struct {
   527→		level    *Level
   528→		hostDir  string
   529→		selector []string
   530→	}
   531→
   532→	n, errs := int32(0), int32(0)
   533→	var successDirs []string
   534→	var successMu sync.Mutex
   535→
   536→	var wg sync.WaitGroup
   537→	wg.Add(Keys.NumWorkers)
   538→	work := make(chan workItem, Keys.NumWorkers*2)
   539→
   540→	for range Keys.NumWorkers {
   541→		go func() {
   542→			defer wg.Done()
   543→			for wi := range work {
   544→				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
   545→				if err != nil {
   546→					if err == ErrNoNewArchiveData {
   547→						continue
   548→					}
   549→					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
   550→					atomic.AddInt32(&errs, 1)
   551→				} else {
   552→					atomic.AddInt32(&n, 1)
   553→					successMu.Lock()
   554→					successDirs = append(successDirs, wi.hostDir)
   555→					successMu.Unlock()
   556→				}
   557→			}
   558→		}()
   559→	}
   560→
   561→	for i := range levels {
   562→		hostDir := path.Join(dir, path.Join(selectors[i]...))
   563→		work <- workItem{
   564→			level:    levels[i],
   565→			hostDir:  hostDir,
   566→			selector: selectors[i],
   567→		}
   568→	}
   569→	close(work)
   570→	wg.Wait()
   571→
   572→	if errs > 0 {
   573→		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
   574→	}
   575→	return int(n), successDirs, nil
   576→}
   577→
   578→// toCheckpointBinary writes a binary snapshot file for a single host-level node.
   579→// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
   580→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   581→	cf, err := l.toCheckpointFile(from, to, m)
   582→	if err != nil {
   583→		return err
   584→	}
   585→	if cf == nil {
   586→		return ErrNoNewArchiveData
   587→	}
   588→
   589→	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
   590→		return fmt.Errorf("mkdir %s: %w", dir, err)
   591→	}
   592→
   593→	// Write to a temp file first, then rename (atomic on POSIX).
   594→	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
   595→	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))
   596→
   597→	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   598→	if err != nil {
   599→		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
   600→	}
   601→
   602→	bw := bufio.NewWriter(f)
   603→	if err := writeBinarySnapshotFile(bw, cf); err != nil {
   604→		f.Close()
   605→		os.Remove(tmpPath)
   606→		return fmt.Errorf("write binary snapshot: %w", err)
   607→	}
   608→	if err := bw.Flush(); err != nil {
   609→		f.Close()
   610→		os.Remove(tmpPath)
   611→		return err
   612→	}
   613→	f.Close()
   614→
   615→	return os.Rename(tmpPath, finalPath)
   616→}
   617→
   618→// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
   619→func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
   620→	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
   621→		return err
   622→	}
   623→	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
   624→		return err
   625→	}
   626→	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
   627→		return err
   628→	}
   629→	return writeBinaryLevel(w, cf)
   630→}
   631→
   632→// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
   633→func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
   634→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
   635→		return err
   636→	}
   637→
   638→	for name, metric := range cf.Metrics {
   639→		if err := writeString16(w, name); err != nil {
   640→			return err
   641→		}
   642→		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
   643→			return err
   644→		}
   645→		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
   646→			return err
   647→		}
   648→		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
   649→			return err
   650→		}
   651→		for _, v := range metric.Data {
   652→			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
   653→				return err
   654→			}
   655→		}
   656→	}
   657→
   658→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
   659→		return err
   660→	}
   661→
   662→	for name, child := range cf.Children {
   663→		if err := writeString16(w, name); err != nil {
   664→			return err
   665→		}
   666→		if err := writeBinaryLevel(w, child); err != nil {
   667→			return err
   668→		}
   669→	}
   670→
   671→	return nil
   672→}
   673→
   674→// writeString16 writes a 2-byte length-prefixed string to w.
   675→func writeString16(w io.Writer, s string) error {
   676→	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
   677→		return err
   678→	}
   679→	_, err := io.WriteString(w, s)
   680→	return err
   681→}
   682→
   683→// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
   684→// The retention check (from) is applied to the file's 'to' timestamp.
   685→func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
   686→	br := bufio.NewReader(f)
   687→
   688→	var magic uint32
   689→	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
   690→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
   691→	}
   692→	if magic != snapFileMagic {
   693→		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
   694→	}
   695→
   696→	var fileFrom, fileTo int64
   697→	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
   698→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
   699→	}
   700→	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
   701→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
   702→	}
   703→
   704→	if fileTo != 0 && fileTo < from {
   705→		return nil // File is older than retention window, skip it
   706→	}
   707→
   708→	cf, err := readBinaryLevel(br)
   709→	if err != nil {
   710→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
   711→	}
   712→	cf.From = fileFrom
   713→	cf.To = fileTo
   714→
   715→	return l.loadFile(cf, m)
   716→}
   717→
   718→// readBinaryLevel recursively reads a level from the binary snapshot format.
   719→func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
   720→	cf := &CheckpointFile{
   721→		Metrics:  make(map[string]*CheckpointMetrics),
   722→		Children: make(map[string]*CheckpointFile),
   723→	}
   724→
   725→	var numMetrics uint32
   726→	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
   727→		return nil, fmt.Errorf("read num_metrics: %w", err)
   728→	}
   729→
   730→	for range numMetrics {
   731→		name, err := readString16(r)
   732→		if err != nil {
   733→			return nil, fmt.Errorf("read metric name: %w", err)
   734→		}
   735→
   736→		var freq, start int64
   737→		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
   738→			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
   739→		}
   740→		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
   741→			return nil, fmt.Errorf("read start for %s: %w", name, err)
   742→		}
   743→
   744→		var numValues uint32
   745→		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
   746→			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
   747→		}
   748→
   749→		data := make([]schema.Float, numValues)
   750→		for i := range numValues {
   751→			var bits uint32
   752→			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
   753→				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
   754→			}
   755→			data[i] = schema.Float(math.Float32frombits(bits))
   756→		}
   757→
   758→		cf.Metrics[name] = &CheckpointMetrics{
   759→			Frequency: freq,
   760→			Start:     start,
   761→			Data:      data,
   762→		}
   763→	}
   764→
   765→	var numChildren uint32
   766→	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
   767→		return nil, fmt.Errorf("read num_children: %w", err)
   768→	}
   769→
   770→	for range numChildren {
   771→		childName, err := readString16(r)
   772→		if err != nil {
   773→			return nil, fmt.Errorf("read child name: %w", err)
   774→		}
   775→
   776→		child, err := readBinaryLevel(r)
   777→		if err != nil {
   778→			return nil, fmt.Errorf("read child %s: %w", childName, err)
   779→		}
   780→		cf.Children[childName] = child
   781→	}
   782→
   783→	return cf, nil
   784→}
   785→
   786→// readString16 reads a 2-byte length-prefixed string from r.
   787→func readString16(r io.Reader) (string, error) {
   788→	var sLen uint16
   789→	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
   790→		return "", err
   791→	}
   792→	buf := make([]byte, sLen)
   793→	if _, err := io.ReadFull(r, buf); err != nil {
   794→		return "", err
   795→	}
   796→	return string(buf), nil
   797→}
   798→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼
//	        WALStaging goroutine
//	               │
//	               ▼
//	        checkpoints/cluster/host/current.wal  (append-only, binary)
//
//	Periodic checkpoint (Checkpointing goroutine):
//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
//	  2. Signal WALStaging to truncate current.wal per host
//
//	On restart (FromCheckpoint):
//	  1. Load most recent <timestamp>.bin snapshot
//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
//
// # WAL Record Format
//
//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
//
//	payload:
//	  [8B timestamp int64]
//	  [2B metric_name_len uint16][N metric name bytes]
//	  [1B selector_count uint8]
//	  per selector: [1B selector_len uint8][M selector bytes]
//	  [4B value float32 bits]
//
// # Binary Snapshot Format
//
//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
//	Level tree (recursive):
//	  [4B num_metrics uint32]
//	  per metric:
//	    [2B name_len uint16][N name bytes]
//	    [8B frequency int64][8B start int64]
//	    [4B num_values uint32][num_values × 4B float32]
//	  [4B num_children uint32]
//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
package metricstore

import (
	"bufio"
	"context"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"io"
	"math"
	"os"
	"path"
	"strings"
	"sync"
	"sync/atomic"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Magic numbers for binary formats.
const (
	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
)

// WALMessages is the channel for sending metric writes to the WAL staging goroutine.
// Buffered to allow burst writes without blocking the metric ingestion path.
var WALMessages = make(chan *WALMessage, 4096)

// walRotateCh is used by the checkpoint goroutine to request WAL file rotation
// (close, delete, reopen) after a binary snapshot has been written.
var walRotateCh = make(chan walRotateReq, 256)

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle for one host directory.
type walFileState struct {
	f *os.File
}

// WALStaging starts a background goroutine that receives WALMessage items
// and appends binary WAL records to per-host current.wal files.
// Also handles WAL rotation requests from the checkpoint goroutine.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	wg.Go(func() {
		if Keys.Checkpoints.FileFormat == "json" {
			return
		}

		hostFiles := make(map[string]*walFileState)

		defer func() {
			for _, ws := range hostFiles {
				if ws.f != nil {
					ws.f.Close()
				}
			}
		}()

		getOrOpenWAL := func(hostDir string) *os.File {
			ws, ok := hostFiles[hostDir]
			if ok {
				return ws.f
			}

			if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
				cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
				return nil
			}

			walPath := path.Join(hostDir, "current.wal")
			f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
			if err != nil {
				cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
				return nil
			}

			// Write file header magic if file is new (empty).
			info, err := f.Stat()
			if err == nil && info.Size() == 0 {
				var hdr [4]byte
				binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
				if _, err := f.Write(hdr[:]); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
					f.Close()
					return nil
				}
			}

			hostFiles[hostDir] = &walFileState{f: f}
			return f
		}

		processMsg := func(msg *WALMessage) {
			hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
			f := getOrOpenWAL(hostDir)
			if f == nil {
				return
			}
			if err := writeWALRecord(f, msg); err != nil {
				cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
			}
		}

		processRotate := func(req walRotateReq) {
			ws, ok := hostFiles[req.hostDir]
			if ok && ws.f != nil {
				ws.f.Close()
				walPath := path.Join(req.hostDir, "current.wal")
				if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
					cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
				}
				delete(hostFiles, req.hostDir)
			}
			close(req.done)
		}

		drain := func() {
			for {
				select {
				case msg, ok := <-WALMessages:
					if !ok {
						return
					}
					processMsg(msg)
				case req := <-walRotateCh:
					processRotate(req)
				default:
					return
				}
			}
		}

		for {
			select {
			case <-ctx.Done():
				drain()
				return
			case msg, ok := <-WALMessages:
				if !ok {
					return
				}
				processMsg(msg)
			case req := <-walRotateCh:
				processRotate(req)
			}
		}
	})
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete.
func RotateWALFiles(hostDirs []string) {
	dones := make([]chan struct{}, len(hostDirs))
	for i, dir := range hostDirs {
		dones[i] = make(chan struct{})
		walRotateCh <- walRotateReq{hostDir: dir, done: dones[i]}
	}
	for _, done := range dones {
		<-done
	}
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete.
func RotateWALFilesAfterShutdown(hostDirs []string) {
	for _, dir := range hostDirs {
		walPath := path.Join(dir, "current.wal")
		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
		}
	}
}

// buildWALPayload encodes a WALMessage into a binary payload (without magic/length/CRC).
func buildWALPayload(msg *WALMessage) []byte {
	size := 8 + 2 + len(msg.MetricName) + 1 + 4
	for _, s := range msg.Selector {
		size += 1 + len(s)
	}

	buf := make([]byte, 0, size)

	// Timestamp (8 bytes, little-endian int64)
	var ts [8]byte
	binary.LittleEndian.PutUint64(ts[:], uint64(msg.Timestamp))
	buf = append(buf, ts[:]...)

	// Metric name (2-byte length prefix + bytes)
	var mLen [2]byte
	binary.LittleEndian.PutUint16(mLen[:], uint16(len(msg.MetricName)))
	buf = append(buf, mLen[:]...)
	buf = append(buf, msg.MetricName...)

	// Selector count (1 byte)
	buf = append(buf, byte(len(msg.Selector)))

	// Selectors (1-byte length prefix + bytes each)
	for _, sel := range msg.Selector {
		buf = append(buf, byte(len(sel)))
		buf = append(buf, sel...)
	}

	// Value (4 bytes, float32 bit representation)
	var val [4]byte
	binary.LittleEndian.PutUint32(val[:], math.Float32bits(float32(msg.Value)))
	buf = append(buf, val[:]...)

	return buf
}

// writeWALRecord appends a binary WAL record to the file.
// Format: [4B magic][4B payload_len][payload][4B CRC32]
func writeWALRecord(f *os.File, msg *WALMessage) error {
	payload := buildWALPayload(msg)
	crc := crc32.ChecksumIEEE(payload)

	record := make([]byte, 0, 4+4+len(payload)+4)

	var magic [4]byte
	binary.LittleEndian.PutUint32(magic[:], walRecordMagic)
	record = append(record, magic[:]...)

	var pLen [4]byte
	binary.LittleEndian.PutUint32(pLen[:], uint32(len(payload)))
	record = append(record, pLen[:]...)

	record = append(record, payload...)

	var crcBytes [4]byte
	binary.LittleEndian.PutUint32(crcBytes[:], crc)
	record = append(record, crcBytes[:]...)

	_, err := f.Write(record)
	return err
}

// readWALRecord reads one WAL record from the reader.
// Returns (nil, nil) on clean EOF. Returns error on data corruption.
// A CRC mismatch indicates a truncated trailing record (expected on crash).
func readWALRecord(r io.Reader) (*WALMessage, error) {
	var magic uint32
	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
		if err == io.EOF {
			return nil, nil // Clean EOF
		}
		return nil, fmt.Errorf("read record magic: %w", err)
	}

	if magic != walRecordMagic {
		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
	}

	var payloadLen uint32
	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
		return nil, fmt.Errorf("read payload length: %w", err)
	}

	if payloadLen > 1<<20 { // 1 MB sanity limit
		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
	}

	payload := make([]byte, payloadLen)
	if _, err := io.ReadFull(r, payload); err != nil {
		return nil, fmt.Errorf("read payload: %w", err)
	}

	var storedCRC uint32
	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
		return nil, fmt.Errorf("read CRC: %w", err)
	}

	if crc32.ChecksumIEEE(payload) != storedCRC {
		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
	}

	return parseWALPayload(payload)
}

// parseWALPayload decodes a binary payload into a WALMessage.
func parseWALPayload(payload []byte) (*WALMessage, error) {
	if len(payload) < 8+2+1+4 {
		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
	}

	offset := 0

	// Timestamp (8 bytes)
	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
	offset += 8

	// Metric name (2-byte length + bytes)
	if offset+2 > len(payload) {
		return nil, fmt.Errorf("metric name length overflows payload")
	}
	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
	offset += 2

	if offset+mLen > len(payload) {
		return nil, fmt.Errorf("metric name overflows payload")
	}
	metricName := string(payload[offset : offset+mLen])
	offset += mLen

	// Selector count (1 byte)
	if offset >= len(payload) {
		return nil, fmt.Errorf("selector count overflows payload")
	}
	selCount := int(payload[offset])
	offset++

	selectors := make([]string, selCount)
	for i := range selCount {
		if offset >= len(payload) {
			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
		}
		sLen := int(payload[offset])
		offset++

		if offset+sLen > len(payload) {
			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
		}
		selectors[i] = string(payload[offset : offset+sLen])
		offset += sLen
	}

	// Value (4 bytes, float32 bits)
	if offset+4 > len(payload) {
		return nil, fmt.Errorf("value overflows payload")
	}
	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
	value := schema.Float(math.Float32frombits(bits))

	return &WALMessage{
		MetricName: metricName,
		Timestamp:  ts,
		Selector:   selectors,
		Value:      value,
	}, nil
}

// loadWALFile reads a WAL file and replays all valid records into the Level tree.
// l is the host-level node. Corrupt or partial trailing records are silently skipped
// (expected on crash). Records older than 'from' are skipped.
func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	// Verify file header magic.
	var fileMagic uint32
	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
		if err == io.EOF {
			return nil // Empty file, no data
		}
		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
	}

	if fileMagic != walFileMagic {
		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
	}

	// Cache level lookups to avoid repeated tree traversal.
	lvlCache := make(map[string]*Level)

	for {
		msg, err := readWALRecord(br)
		if err != nil {
			// Truncated trailing record is expected after a crash; stop replaying.
			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
			break
		}
		if msg == nil {
			break // Clean EOF
		}

		if msg.Timestamp < from {
			continue // Older than retention window
		}

		minfo, ok := m.Metrics[msg.MetricName]
		if !ok {
			continue // Unknown metric (config may have changed)
		}

		// Cache key is the null-separated selector path.
		cacheKey := joinSelector(msg.Selector)
		lvl, ok := lvlCache[cacheKey]
		if !ok {
			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
			lvlCache[cacheKey] = lvl
		}

		// Write directly to the buffer, same as WriteToLevel but without the
		// global level lookup (we already have the right level).
		lvl.lock.Lock()
		b := lvl.metrics[minfo.offset]
		if b == nil {
			b = newBuffer(msg.Timestamp, minfo.Frequency)
			lvl.metrics[minfo.offset] = b
		}
		nb, writeErr := b.write(msg.Timestamp, msg.Value)
		if writeErr == nil && b != nb {
			lvl.metrics[minfo.offset] = nb
		}
		// Ignore write errors for timestamps before buffer start (can happen when
		// replaying WAL entries that predate a loaded snapshot's start time).
		lvl.lock.Unlock()
	}

	return nil
}

// joinSelector builds a cache key from a selector slice using null bytes as separators.
func joinSelector(sel []string) string {
	if len(sel) == 0 {
		return ""
	}
	var result strings.Builder
	result.WriteString(sel[0])
	for i := 1; i < len(sel); i++ {
		result.WriteString("\x00" + sel[i])
	}
	return result.String()
}

// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
// Returns the number of files written, the list of host directories that were
// successfully checkpointed (for WAL rotation), and any errors.
func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
	// Collect all cluster/host pairs.
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		hostDir  string
		selector []string
	}

	n, errs := int32(0), int32(0)
	var successDirs []string
	var successMu sync.Mutex

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)

	for range Keys.NumWorkers {
		go func() {
			defer wg.Done()
			for wi := range work {
				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
				if err != nil {
					if err == ErrNoNewArchiveData {
						continue
					}
					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
					successMu.Lock()
					successDirs = append(successDirs, wi.hostDir)
					successMu.Unlock()
				}
			}
		}()
	}

	for i := range levels {
		hostDir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			hostDir:  hostDir,
			selector: selectors[i],
		}
	}
	close(work)
	wg.Wait()

	if errs > 0 {
		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
	}
	return int(n), successDirs, nil
}

// toCheckpointBinary writes a binary snapshot file for a single host-level node.
// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}
	if cf == nil {
		return ErrNoNewArchiveData
	}

	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
		return fmt.Errorf("mkdir %s: %w", dir, err)
	}

	// Write to a temp file first, then rename (atomic on POSIX).
	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))

	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
	}

	bw := bufio.NewWriter(f)
	if err := writeBinarySnapshotFile(bw, cf); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return fmt.Errorf("write binary snapshot: %w", err)
	}
	if err := bw.Flush(); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return err
	}
	f.Close()

	return os.Rename(tmpPath, finalPath)
}

// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
		return err
	}
	return writeBinaryLevel(w, cf)
}

// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
		return err
	}

	for name, metric := range cf.Metrics {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
			return err
		}
		for _, v := range metric.Data {
			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
				return err
			}
		}
	}

	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
		return err
	}

	for name, child := range cf.Children {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := writeBinaryLevel(w, child); err != nil {
			return err
		}
	}

	return nil
}

// writeString16 writes a 2-byte length-prefixed string to w.
func writeString16(w io.Writer, s string) error {
	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
		return err
	}
	_, err := io.WriteString(w, s)
	return err
}

// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
// The retention check (from) is applied to the file's 'to' timestamp.
func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	var magic uint32
	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
	}
	if magic != snapFileMagic {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
	}

	var fileFrom, fileTo int64
	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
	}
	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
	}

	if fileTo != 0 && fileTo < from {
		return nil // File is older than retention window, skip it
	}

	cf, err := readBinaryLevel(br)
	if err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
	}
	cf.From = fileFrom
	cf.To = fileTo

	return l.loadFile(cf, m)
}

// readBinaryLevel recursively reads a level from the binary snapshot format.
func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
	cf := &CheckpointFile{
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	var numMetrics uint32
	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
		return nil, fmt.Errorf("read num_metrics: %w", err)
	}

	for range numMetrics {
		name, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read metric name: %w", err)
		}

		var freq, start int64
		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
		}
		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
			return nil, fmt.Errorf("read start for %s: %w", name, err)
		}

		var numValues uint32
		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
		}

		data := make([]schema.Float, numValues)
		for i := range numValues {
			var bits uint32
			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
			}
			data[i] = schema.Float(math.Float32frombits(bits))
		}

		cf.Metrics[name] = &CheckpointMetrics{
			Frequency: freq,
			Start:     start,
			Data:      data,
		}
	}

	var numChildren uint32
	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
		return nil, fmt.Errorf("read num_children: %w", err)
	}

	for range numChildren {
		childName, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read child name: %w", err)
		}

		child, err := readBinaryLevel(r)
		if err != nil {
			return nil, fmt.Errorf("read child %s: %w", childName, err)
		}
		cf.Children[childName] = child
	}

	return cf, nil
}

// readString16 reads a 2-byte length-prefixed string from r.
func readString16(r io.Reader) (string, error) {
	var sLen uint16
	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
		return "", err
	}
	buf := make([]byte, sLen)
	if _, err := io.ReadFull(r, buf); err != nil {
		return "", err
	}
	return string(buf), nil
}


   110→// Loaded from cc-backend's config.json "metricstore" section. Controls memory usage,
   111→// persistence, archiving, and metric ingestion.
   112→//
   113→// Fields:
   114→//   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
   115→//   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
   116→//   - MemoryCap:         Max bytes for buffer data (0 = unlimited); triggers forceFree when exceeded
   117→//   - Checkpoints:       Periodic persistence configuration
   118→//   - Debug:             Development/profiling options (nil = disabled)
   119→//   - Archive:           Long-term storage configuration (nil = disabled)
   120→//   - Subscriptions:     NATS topics for metric ingestion (nil = polling only)
   121→type MetricStoreConfig struct {
   122→	// Number of concurrent workers for checkpoint and archive operations.
   123→	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
   124→	NumWorkers         int            `json:"num-workers"`

// Loaded from cc-backend's config.json "metricstore" section. Controls memory usage,
// persistence, archiving, and metric ingestion.
//
// Fields:
//   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
//   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
//   - MemoryCap:         Max bytes for buffer data (0 = unlimited); triggers forceFree when exceeded
//   - Checkpoints:       Periodic persistence configuration
//   - Debug:             Development/profiling options (nil = disabled)
//   - Archive:           Long-term storage configuration (nil = disabled)
//   - Subscriptions:     NATS topics for metric ingestion (nil = polling only)
type MetricStoreConfig struct {
	// Number of concurrent workers for checkpoint and archive operations.
	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
	NumWorkers         int            `json:"num-workers"`


Now I'll implement both improvements. Let me start with all the edits.

**Improvement 1: WAL buffering** — Add `bufio.Writer` to `walFileState` and use it throughout.

**Improvement 3: Fix MemoryCap comment.**


Edit

// walFileState holds an open WAL file handle for one host directory.
type walFileState struct {
	f *os.File
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f *os.File
	w *bufio.Writer
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go has been updated successfully.

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f *os.File
	w *bufio.Writer
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼
//	        WALStaging goroutine
//	               │
//	               ▼
//	        checkpoints/cluster/host/current.wal  (append-only, binary)
//
//	Periodic checkpoint (Checkpointing goroutine):
//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
//	  2. Signal WALStaging to truncate current.wal per host
//
//	On restart (FromCheckpoint):
//	  1. Load most recent <timestamp>.bin snapshot
//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
//
// # WAL Record Format
//
//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
//
//	payload:
//	  [8B timestamp int64]
//	  [2B metric_name_len uint16][N metric name bytes]
//	  [1B selector_count uint8]
//	  per selector: [1B selector_len uint8][M selector bytes]
//	  [4B value float32 bits]
//
// # Binary Snapshot Format
//
//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
//	Level tree (recursive):
//	  [4B num_metrics uint32]
//	  per metric:
//	    [2B name_len uint16][N name bytes]
//	    [8B frequency int64][8B start int64]
//	    [4B num_values uint32][num_values × 4B float32]
//	  [4B num_children uint32]
//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
package metricstore

import (
	"bufio"
	"context"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"io"
	"math"
	"os"
	"path"
	"strings"
	"sync"
	"sync/atomic"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Magic numbers for binary formats.
const (
	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
)

// WALMessages is the channel for sending metric writes to the WAL staging goroutine.
// Buffered to allow burst writes without blocking the metric ingestion path.
var WALMessages = make(chan *WALMessage, 4096)

// walRotateCh is used by the checkpoint goroutine to request WAL file rotation
// (close, delete, reopen) after a binary snapshot has been written.
var walRotateCh = make(chan walRotateReq, 256)

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle for one host directory.
type walFileState struct {
	f *os.File
}

// WALStaging starts a background goroutine that receives WALMessage items
// and appends binary WAL records to per-host current.wal files.
// Also handles WAL rotation requests from the checkpoint goroutine.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	wg.Go(func() {
		if Keys.Checkpoints.FileFormat == "json" {
			return
		}

		hostFiles := make(map[string]*walFileState)

		defer func() {
			for _, ws := range hostFiles {
				if ws.f != nil {
					ws.f.Close()
				}
			}
		}()

		getOrOpenWAL := func(hostDir string) *os.File {
			ws, ok := hostFiles[hostDir]
			if ok {
				return ws.f
			}

			if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
				cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
				return nil
			}

			walPath := path.Join(hostDir, "current.wal")
			f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
			if err != nil {
				cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
				return nil
			}

			// Write file header magic if file is new (empty).
			info, err := f.Stat()
			if err == nil && info.Size() == 0 {
				var hdr [4]byte
				binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
				if _, err := f.Write(hdr[:]); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
					f.Close()
					return nil
				}
			}

			hostFiles[hostDir] = &walFileState{f: f}
			return f
		}

		processMsg := func(msg *WALMessage) {
			hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
			f := getOrOpenWAL(hostDir)
			if f == nil {
				return
			}
			if err := writeWALRecord(f, msg); err != nil {
				cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
			}
		}

		processRotate := func(req walRotateReq) {
			ws, ok := hostFiles[req.hostDir]
			if ok && ws.f != nil {
				ws.f.Close()
				walPath := path.Join(req.hostDir, "current.wal")
				if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
					cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
				}
				delete(hostFiles, req.hostDir)
			}
			close(req.done)
		}

		drain := func() {
			for {
				select {
				case msg, ok := <-WALMessages:
					if !ok {
						return
					}
					processMsg(msg)
				case req := <-walRotateCh:
					processRotate(req)
				default:
					return
				}
			}
		}

		for {
			select {
			case <-ctx.Done():
				drain()
				return
			case msg, ok := <-WALMessages:
				if !ok {
					return
				}
				processMsg(msg)
			case req := <-walRotateCh:
				processRotate(req)
			}
		}
	})
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete.
func RotateWALFiles(hostDirs []string) {
	dones := make([]chan struct{}, len(hostDirs))
	for i, dir := range hostDirs {
		dones[i] = make(chan struct{})
		walRotateCh <- walRotateReq{hostDir: dir, done: dones[i]}
	}
	for _, done := range dones {
		<-done
	}
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete.
func RotateWALFilesAfterShutdown(hostDirs []string) {
	for _, dir := range hostDirs {
		walPath := path.Join(dir, "current.wal")
		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
		}
	}
}

// buildWALPayload encodes a WALMessage into a binary payload (without magic/length/CRC).
func buildWALPayload(msg *WALMessage) []byte {
	size := 8 + 2 + len(msg.MetricName) + 1 + 4
	for _, s := range msg.Selector {
		size += 1 + len(s)
	}

	buf := make([]byte, 0, size)

	// Timestamp (8 bytes, little-endian int64)
	var ts [8]byte
	binary.LittleEndian.PutUint64(ts[:], uint64(msg.Timestamp))
	buf = append(buf, ts[:]...)

	// Metric name (2-byte length prefix + bytes)
	var mLen [2]byte
	binary.LittleEndian.PutUint16(mLen[:], uint16(len(msg.MetricName)))
	buf = append(buf, mLen[:]...)
	buf = append(buf, msg.MetricName...)

	// Selector count (1 byte)
	buf = append(buf, byte(len(msg.Selector)))

	// Selectors (1-byte length prefix + bytes each)
	for _, sel := range msg.Selector {
		buf = append(buf, byte(len(sel)))
		buf = append(buf, sel...)
	}

	// Value (4 bytes, float32 bit representation)
	var val [4]byte
	binary.LittleEndian.PutUint32(val[:], math.Float32bits(float32(msg.Value)))
	buf = append(buf, val[:]...)

	return buf
}

// writeWALRecord appends a binary WAL record to the file.
// Format: [4B magic][4B payload_len][payload][4B CRC32]
func writeWALRecord(f *os.File, msg *WALMessage) error {
	payload := buildWALPayload(msg)
	crc := crc32.ChecksumIEEE(payload)

	record := make([]byte, 0, 4+4+len(payload)+4)

	var magic [4]byte
	binary.LittleEndian.PutUint32(magic[:], walRecordMagic)
	record = append(record, magic[:]...)

	var pLen [4]byte
	binary.LittleEndian.PutUint32(pLen[:], uint32(len(payload)))
	record = append(record, pLen[:]...)

	record = append(record, payload...)

	var crcBytes [4]byte
	binary.LittleEndian.PutUint32(crcBytes[:], crc)
	record = append(record, crcBytes[:]...)

	_, err := f.Write(record)
	return err
}

// readWALRecord reads one WAL record from the reader.
// Returns (nil, nil) on clean EOF. Returns error on data corruption.
// A CRC mismatch indicates a truncated trailing record (expected on crash).
func readWALRecord(r io.Reader) (*WALMessage, error) {
	var magic uint32
	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
		if err == io.EOF {
			return nil, nil // Clean EOF
		}
		return nil, fmt.Errorf("read record magic: %w", err)
	}

	if magic != walRecordMagic {
		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
	}

	var payloadLen uint32
	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
		return nil, fmt.Errorf("read payload length: %w", err)
	}

	if payloadLen > 1<<20 { // 1 MB sanity limit
		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
	}

	payload := make([]byte, payloadLen)
	if _, err := io.ReadFull(r, payload); err != nil {
		return nil, fmt.Errorf("read payload: %w", err)
	}

	var storedCRC uint32
	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
		return nil, fmt.Errorf("read CRC: %w", err)
	}

	if crc32.ChecksumIEEE(payload) != storedCRC {
		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
	}

	return parseWALPayload(payload)
}

// parseWALPayload decodes a binary payload into a WALMessage.
func parseWALPayload(payload []byte) (*WALMessage, error) {
	if len(payload) < 8+2+1+4 {
		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
	}

	offset := 0

	// Timestamp (8 bytes)
	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
	offset += 8

	// Metric name (2-byte length + bytes)
	if offset+2 > len(payload) {
		return nil, fmt.Errorf("metric name length overflows payload")
	}
	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
	offset += 2

	if offset+mLen > len(payload) {
		return nil, fmt.Errorf("metric name overflows payload")
	}
	metricName := string(payload[offset : offset+mLen])
	offset += mLen

	// Selector count (1 byte)
	if offset >= len(payload) {
		return nil, fmt.Errorf("selector count overflows payload")
	}
	selCount := int(payload[offset])
	offset++

	selectors := make([]string, selCount)
	for i := range selCount {
		if offset >= len(payload) {
			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
		}
		sLen := int(payload[offset])
		offset++

		if offset+sLen > len(payload) {
			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
		}
		selectors[i] = string(payload[offset : offset+sLen])
		offset += sLen
	}

	// Value (4 bytes, float32 bits)
	if offset+4 > len(payload) {
		return nil, fmt.Errorf("value overflows payload")
	}
	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
	value := schema.Float(math.Float32frombits(bits))

	return &WALMessage{
		MetricName: metricName,
		Timestamp:  ts,
		Selector:   selectors,
		Value:      value,
	}, nil
}

// loadWALFile reads a WAL file and replays all valid records into the Level tree.
// l is the host-level node. Corrupt or partial trailing records are silently skipped
// (expected on crash). Records older than 'from' are skipped.
func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	// Verify file header magic.
	var fileMagic uint32
	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
		if err == io.EOF {
			return nil // Empty file, no data
		}
		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
	}

	if fileMagic != walFileMagic {
		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
	}

	// Cache level lookups to avoid repeated tree traversal.
	lvlCache := make(map[string]*Level)

	for {
		msg, err := readWALRecord(br)
		if err != nil {
			// Truncated trailing record is expected after a crash; stop replaying.
			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
			break
		}
		if msg == nil {
			break // Clean EOF
		}

		if msg.Timestamp < from {
			continue // Older than retention window
		}

		minfo, ok := m.Metrics[msg.MetricName]
		if !ok {
			continue // Unknown metric (config may have changed)
		}

		// Cache key is the null-separated selector path.
		cacheKey := joinSelector(msg.Selector)
		lvl, ok := lvlCache[cacheKey]
		if !ok {
			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
			lvlCache[cacheKey] = lvl
		}

		// Write directly to the buffer, same as WriteToLevel but without the
		// global level lookup (we already have the right level).
		lvl.lock.Lock()
		b := lvl.metrics[minfo.offset]
		if b == nil {
			b = newBuffer(msg.Timestamp, minfo.Frequency)
			lvl.metrics[minfo.offset] = b
		}
		nb, writeErr := b.write(msg.Timestamp, msg.Value)
		if writeErr == nil && b != nb {
			lvl.metrics[minfo.offset] = nb
		}
		// Ignore write errors for timestamps before buffer start (can happen when
		// replaying WAL entries that predate a loaded snapshot's start time).
		lvl.lock.Unlock()
	}

	return nil
}

// joinSelector builds a cache key from a selector slice using null bytes as separators.
func joinSelector(sel []string) string {
	if len(sel) == 0 {
		return ""
	}
	var result strings.Builder
	result.WriteString(sel[0])
	for i := 1; i < len(sel); i++ {
		result.WriteString("\x00" + sel[i])
	}
	return result.String()
}

// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
// Returns the number of files written, the list of host directories that were
// successfully checkpointed (for WAL rotation), and any errors.
func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
	// Collect all cluster/host pairs.
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		hostDir  string
		selector []string
	}

	n, errs := int32(0), int32(0)
	var successDirs []string
	var successMu sync.Mutex

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)

	for range Keys.NumWorkers {
		go func() {
			defer wg.Done()
			for wi := range work {
				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
				if err != nil {
					if err == ErrNoNewArchiveData {
						continue
					}
					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
					successMu.Lock()
					successDirs = append(successDirs, wi.hostDir)
					successMu.Unlock()
				}
			}
		}()
	}

	for i := range levels {
		hostDir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			hostDir:  hostDir,
			selector: selectors[i],
		}
	}
	close(work)
	wg.Wait()

	if errs > 0 {
		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
	}
	return int(n), successDirs, nil
}

// toCheckpointBinary writes a binary snapshot file for a single host-level node.
// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}
	if cf == nil {
		return ErrNoNewArchiveData
	}

	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
		return fmt.Errorf("mkdir %s: %w", dir, err)
	}

	// Write to a temp file first, then rename (atomic on POSIX).
	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))

	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
	}

	bw := bufio.NewWriter(f)
	if err := writeBinarySnapshotFile(bw, cf); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return fmt.Errorf("write binary snapshot: %w", err)
	}
	if err := bw.Flush(); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return err
	}
	f.Close()

	return os.Rename(tmpPath, finalPath)
}

// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
		return err
	}
	return writeBinaryLevel(w, cf)
}

// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
		return err
	}

	for name, metric := range cf.Metrics {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
			return err
		}
		for _, v := range metric.Data {
			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
				return err
			}
		}
	}

	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
		return err
	}

	for name, child := range cf.Children {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := writeBinaryLevel(w, child); err != nil {
			return err
		}
	}

	return nil
}

// writeString16 writes a 2-byte length-prefixed string to w.
func writeString16(w io.Writer, s string) error {
	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
		return err
	}
	_, err := io.WriteString(w, s)
	return err
}

// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
// The retention check (from) is applied to the file's 'to' timestamp.
func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	var magic uint32
	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
	}
	if magic != snapFileMagic {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
	}

	var fileFrom, fileTo int64
	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
	}
	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
	}

	if fileTo != 0 && fileTo < from {
		return nil // File is older than retention window, skip it
	}

	cf, err := readBinaryLevel(br)
	if err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
	}
	cf.From = fileFrom
	cf.To = fileTo

	return l.loadFile(cf, m)
}

// readBinaryLevel recursively reads a level from the binary snapshot format.
func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
	cf := &CheckpointFile{
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	var numMetrics uint32
	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
		return nil, fmt.Errorf("read num_metrics: %w", err)
	}

	for range numMetrics {
		name, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read metric name: %w", err)
		}

		var freq, start int64
		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
		}
		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
			return nil, fmt.Errorf("read start for %s: %w", name, err)
		}

		var numValues uint32
		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
		}

		data := make([]schema.Float, numValues)
		for i := range numValues {
			var bits uint32
			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
			}
			data[i] = schema.Float(math.Float32frombits(bits))
		}

		cf.Metrics[name] = &CheckpointMetrics{
			Frequency: freq,
			Start:     start,
			Data:      data,
		}
	}

	var numChildren uint32
	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
		return nil, fmt.Errorf("read num_children: %w", err)
	}

	for range numChildren {
		childName, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read child name: %w", err)
		}

		child, err := readBinaryLevel(r)
		if err != nil {
			return nil, fmt.Errorf("read child %s: %w", childName, err)
		}
		cf.Children[childName] = child
	}

	return cf, nil
}

// readString16 reads a 2-byte length-prefixed string from r.
func readString16(r io.Reader) (string, error) {
	var sLen uint16
	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
		return "", err
	}
	buf := make([]byte, sLen)
	if _, err := io.ReadFull(r, buf); err != nil {
		return "", err
	}
	return string(buf), nil
}

-// walFileState holds an open WAL file handle for one host directory.

+// walFileState holds an open WAL file handle and buffered writer for one host directory.


		defer func() {
			for _, ws := range hostFiles {
				if ws.f != nil {
					ws.f.Close()
				}
			}
		}()

		defer func() {
			for _, ws := range hostFiles {
				if ws.f != nil {
					ws.w.Flush()
					ws.f.Close()
				}
			}
		}()