	// Single atomic write to the buffered writer.
	n, err := w.Write(buf)
	return n, err
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼
//	        WALStaging goroutine
//	               │
//	               ▼
//	        checkpoints/cluster/host/current.wal  (append-only, binary)
//
//	Periodic checkpoint (Checkpointing goroutine):
//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
//	  2. Signal WALStaging to truncate current.wal per host
//
//	On restart (FromCheckpoint):
//	  1. Load most recent <timestamp>.bin snapshot
//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
//
// # WAL Record Format
//
//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
//
//	payload:
//	  [8B timestamp int64]
//	  [2B metric_name_len uint16][N metric name bytes]
//	  [1B selector_count uint8]
//	  per selector: [1B selector_len uint8][M selector bytes]
//	  [4B value float32 bits]
//
// # Binary Snapshot Format
//
//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
//	Level tree (recursive):
//	  [4B num_metrics uint32]
//	  per metric:
//	    [2B name_len uint16][N name bytes]
//	    [8B frequency int64][8B start int64]
//	    [4B num_values uint32][num_values × 4B float32]
//	  [4B num_children uint32]
//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
package metricstore

import (
	"bufio"
	"context"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"hash/fnv"
	"io"
	"math"
	"os"
	"path"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Magic numbers for binary formats.
const (
	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
)

// walShardChs holds per-shard channels for WAL messages.
// Initialized by WALStaging; nil when WAL is not active.
var walShardChs []chan *WALMessage

// walShardRotateChs holds per-shard channels for WAL rotation requests.
var walShardRotateChs []chan walRotateReq

// walNumShards stores the number of shards (set during WALStaging init).
var walNumShards int

// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
var walStagingWg sync.WaitGroup

// walShuttingDown is set before closing shard channels to prevent
// SendWALMessage from sending on a closed channel (which panics in Go).
var walShuttingDown atomic.Bool

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f     *os.File
	w     *bufio.Writer
	dirty bool
	size  int64 // approximate bytes written (tracked from open + writes)
}

// walFlushInterval controls how often dirty WAL files are flushed to disk.
// Decoupling flushes from message processing lets the consumer run at memory
// speed, amortizing syscall overhead across many writes.
const walFlushInterval = 1 * time.Second

// walShardIndex computes which shard a message belongs to based on cluster+node.
// Uses FNV-1a hash for fast, well-distributed mapping.
func walShardIndex(cluster, node string) int {
	h := fnv.New32a()
	h.Write([]byte(cluster))
	h.Write([]byte{0})
	h.Write([]byte(node))
	return int(h.Sum32()) % walNumShards
}

// SendWALMessage routes a WAL message to the appropriate shard channel.
// Returns false if the channel is full or shutdown is in progress.
func SendWALMessage(msg *WALMessage) bool {
	if walShardChs == nil || walShuttingDown.Load() {
		return false
	}
	shard := walShardIndex(msg.Cluster, msg.Node)
	select {
	case walShardChs[shard] <- msg:
		return true
	default:
		return false
	}
}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.
// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return
	}

	walNumShards = max(Keys.NumWorkers, 1)
	chBufSize := max(65536/walNumShards, 1024)

	walShardChs = make([]chan *WALMessage, walNumShards)
	walShardRotateChs = make([]chan walRotateReq, walNumShards)

	for i := range walNumShards {
		walShardChs[i] = make(chan *WALMessage, chBufSize)
		walShardRotateChs[i] = make(chan walRotateReq, 64)
	}

	for i := range walNumShards {
		msgCh := walShardChs[i]
		rotateCh := walShardRotateChs[i]

		walStagingWg.Add(1)
		wg.Go(func() {
			defer walStagingWg.Done()
			hostFiles := make(map[string]*walFileState)

			defer func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
						ws.f.Close()
					}
				}
			}()

			getOrOpenWAL := func(hostDir string) *walFileState {
				ws, ok := hostFiles[hostDir]
				if ok {
					return ws
				}

				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
					return nil
				}

				walPath := path.Join(hostDir, "current.wal")
				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
					return nil
				}

				w := bufio.NewWriter(f)

				// Write file header magic if file is new (empty).
				info, err := f.Stat()
				var fileSize int64
				if err == nil {
					fileSize = info.Size()
				}
				if err == nil && fileSize == 0 {
					var hdr [4]byte
					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
					if _, err := w.Write(hdr[:]); err != nil {
						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
						f.Close()
						return nil
					}
					fileSize = 4
				}

				ws = &walFileState{f: f, w: w, size: fileSize}
				hostFiles[hostDir] = ws
				return ws
			}

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}
				if err := writeWALRecordDirect(ws.w, msg); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
				ws.dirty = true
			}

			processRotate := func(req walRotateReq) {
				ws, ok := hostFiles[req.hostDir]
				if ok && ws.f != nil {
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(req.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, req.hostDir)
				}
				close(req.done)
			}

			flushDirty := func() {
				for _, ws := range hostFiles {
					if ws.dirty {
						ws.w.Flush()
						ws.dirty = false
					}
				}
			}

			ticker := time.NewTicker(walFlushInterval)
			defer ticker.Stop()

			// drainBatch processes up to 4096 pending messages without blocking.
			// Returns false if the channel was closed.
			drainBatch := func() bool {
				for range 4096 {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							flushDirty()
							return false
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						return true
					}
				}
				return true
			}

			for {
				select {
				case <-ctx.Done():
					// On shutdown, skip draining buffered messages — a full binary
					// checkpoint will be written from in-memory state, making
					// buffered WAL records redundant.
					flushDirty()
					return
				case msg, ok := <-msgCh:
					if !ok {
						return
					}
					processMsg(msg)
					if !drainBatch() {
						return
					}
					// No flush here — timer handles periodic flushing.
				case <-ticker.C:
					flushDirty()
				case req := <-rotateCh:
					processRotate(req)
				}
			}
		})
	}
}

// WaitForWALStagingDrain blocks until all WALStaging goroutines have exited.
// Must be called after closing walShardChs to ensure all file handles are
// flushed and closed before checkpoint writes begin.
func WaitForWALStagingDrain() {
	walStagingWg.Wait()
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete. Each request is routed to the
// shard that owns the host directory.
//
// If shutdown is in progress (WAL staging goroutines may have exited),
// rotation is skipped to avoid deadlocking on abandoned channels.
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		default:
			// Channel full or goroutine not consuming — skip this host.
			cclog.Warnf("[METRICSTORE]> WAL rotation skipped for %s (channel full)", dir)
		}
	}
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation timed out, continuing")
			return
		}
	}
}

// walShardIndexFromDir extracts cluster and node from a host directory path
// and returns the shard index. The path format is: rootDir/cluster/node
func walShardIndexFromDir(hostDir string) int {
	// hostDir = .../cluster/node — extract last two path components
	node := path.Base(hostDir)
	cluster := path.Base(path.Dir(hostDir))
	return walShardIndex(cluster, node)
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete.
func RotateWALFilesAfterShutdown(hostDirs []string) {
	for _, dir := range hostDirs {
		walPath := path.Join(dir, "current.wal")
		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
		}
	}
}

// writeWALRecordDirect encodes a WAL record into a contiguous buffer first,
// then writes it to the bufio.Writer in a single call. This prevents partial
// records in the write buffer if a write error occurs mid-record (e.g. disk full).
// Returns the number of bytes written and any error.
func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) (int, error) {
	// Compute payload and total record size.
	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
	for _, s := range msg.Selector {
		payloadSize += 1 + len(s)
	}
	// Total: 8 (header) + payload + 4 (CRC).
	totalSize := 8 + payloadSize + 4

	// Use stack buffer for typical small records, heap-allocate only for large ones.
	var stackBuf [256]byte
	var buf []byte
	if totalSize <= len(stackBuf) {
		buf = stackBuf[:totalSize]
	} else {
		buf = make([]byte, totalSize)
	}

	// Header: magic + payload length.
	binary.LittleEndian.PutUint32(buf[0:4], walRecordMagic)
	binary.LittleEndian.PutUint32(buf[4:8], uint32(payloadSize))

	// Payload starts at offset 8.
	p := 8

	// Timestamp (8 bytes).
	binary.LittleEndian.PutUint64(buf[p:p+8], uint64(msg.Timestamp))
	p += 8

	// Metric name length (2 bytes) + metric name.
	binary.LittleEndian.PutUint16(buf[p:p+2], uint16(len(msg.MetricName)))
	p += 2
	p += copy(buf[p:], msg.MetricName)

	// Selector count (1 byte).
	buf[p] = byte(len(msg.Selector))
	p++

	// Selectors (1-byte length + bytes each).
	for _, sel := range msg.Selector {
		buf[p] = byte(len(sel))
		p++
		p += copy(buf[p:], sel)
	}

	// Value (4 bytes, float32 bits).
	binary.LittleEndian.PutUint32(buf[p:p+4], math.Float32bits(float32(msg.Value)))
	p += 4

	// CRC32 over payload (bytes 8..8+payloadSize).
	crc := crc32.ChecksumIEEE(buf[8 : 8+payloadSize])
	binary.LittleEndian.PutUint32(buf[p:p+4], crc)

	// Single atomic write to the buffered writer.
	_, err := w.Write(buf)
	return err
}

// readWALRecord reads one WAL record from the reader.
// Returns (nil, nil) on clean EOF. Returns error on data corruption.
// A CRC mismatch indicates a truncated trailing record (expected on crash).
func readWALRecord(r io.Reader) (*WALMessage, error) {
	var magic uint32
	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
		if err == io.EOF {
			return nil, nil // Clean EOF
		}
		return nil, fmt.Errorf("read record magic: %w", err)
	}

	if magic != walRecordMagic {
		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
	}

	var payloadLen uint32
	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
		return nil, fmt.Errorf("read payload length: %w", err)
	}

	if payloadLen > 1<<20 { // 1 MB sanity limit
		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
	}

	payload := make([]byte, payloadLen)
	if _, err := io.ReadFull(r, payload); err != nil {
		return nil, fmt.Errorf("read payload: %w", err)
	}

	var storedCRC uint32
	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
		return nil, fmt.Errorf("read CRC: %w", err)
	}

	if crc32.ChecksumIEEE(payload) != storedCRC {
		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
	}

	return parseWALPayload(payload)
}

// parseWALPayload decodes a binary payload into a WALMessage.
func parseWALPayload(payload []byte) (*WALMessage, error) {
	if len(payload) < 8+2+1+4 {
		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
	}

	offset := 0

	// Timestamp (8 bytes)
	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
	offset += 8

	// Metric name (2-byte length + bytes)
	if offset+2 > len(payload) {
		return nil, fmt.Errorf("metric name length overflows payload")
	}
	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
	offset += 2

	if offset+mLen > len(payload) {
		return nil, fmt.Errorf("metric name overflows payload")
	}
	metricName := string(payload[offset : offset+mLen])
	offset += mLen

	// Selector count (1 byte)
	if offset >= len(payload) {
		return nil, fmt.Errorf("selector count overflows payload")
	}
	selCount := int(payload[offset])
	offset++

	selectors := make([]string, selCount)
	for i := range selCount {
		if offset >= len(payload) {
			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
		}
		sLen := int(payload[offset])
		offset++

		if offset+sLen > len(payload) {
			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
		}
		selectors[i] = string(payload[offset : offset+sLen])
		offset += sLen
	}

	// Value (4 bytes, float32 bits)
	if offset+4 > len(payload) {
		return nil, fmt.Errorf("value overflows payload")
	}
	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
	value := schema.Float(math.Float32frombits(bits))

	return &WALMessage{
		MetricName: metricName,
		Timestamp:  ts,
		Selector:   selectors,
		Value:      value,
	}, nil
}

// loadWALFile reads a WAL file and replays all valid records into the Level tree.
// l is the host-level node. Corrupt or partial trailing records are silently skipped
// (expected on crash). Records older than 'from' are skipped.
func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	// Verify file header magic.
	var fileMagic uint32
	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
		if err == io.EOF {
			return nil // Empty file, no data
		}
		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
	}

	if fileMagic != walFileMagic {
		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
	}

	// Cache level lookups to avoid repeated tree traversal.
	lvlCache := make(map[string]*Level)

	for {
		msg, err := readWALRecord(br)
		if err != nil {
			// Truncated trailing record is expected after a crash; stop replaying.
			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
			break
		}
		if msg == nil {
			break // Clean EOF
		}

		if msg.Timestamp < from {
			continue // Older than retention window
		}

		minfo, ok := m.Metrics[msg.MetricName]
		if !ok {
			continue // Unknown metric (config may have changed)
		}

		// Cache key is the null-separated selector path.
		cacheKey := joinSelector(msg.Selector)
		lvl, ok := lvlCache[cacheKey]
		if !ok {
			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
			lvlCache[cacheKey] = lvl
		}

		// Write directly to the buffer, same as WriteToLevel but without the
		// global level lookup (we already have the right level).
		lvl.lock.Lock()
		b := lvl.metrics[minfo.offset]
		if b == nil {
			b = newBuffer(msg.Timestamp, minfo.Frequency)
			lvl.metrics[minfo.offset] = b
		}
		nb, writeErr := b.write(msg.Timestamp, msg.Value)
		if writeErr == nil && b != nb {
			lvl.metrics[minfo.offset] = nb
		}
		// Ignore write errors for timestamps before buffer start (can happen when
		// replaying WAL entries that predate a loaded snapshot's start time).
		lvl.lock.Unlock()
	}

	return nil
}

// joinSelector builds a cache key from a selector slice using null bytes as separators.
func joinSelector(sel []string) string {
	if len(sel) == 0 {
		return ""
	}
	var result strings.Builder
	result.WriteString(sel[0])
	for i := 1; i < len(sel); i++ {
		result.WriteString("\x00" + sel[i])
	}
	return result.String()
}

// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
// Returns the number of files written, the list of host directories that were
// successfully checkpointed (for WAL rotation), and any errors.
func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
	// Collect all cluster/host pairs.
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		hostDir  string
		selector []string
	}

	totalWork := len(levels)
	cclog.Infof("[METRICSTORE]> Starting binary checkpoint for %d hosts with %d workers", totalWork, Keys.NumWorkers)

	n, errs, completed := int32(0), int32(0), int32(0)
	var successDirs []string
	var successMu sync.Mutex

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)

	// Progress logging goroutine.
	stopProgress := make(chan struct{})
	go func() {
		ticker := time.NewTicker(10 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				cclog.Infof("[METRICSTORE]> Checkpoint progress: %d/%d hosts (%d written, %d errors)",
					atomic.LoadInt32(&completed), totalWork, atomic.LoadInt32(&n), atomic.LoadInt32(&errs))
			case <-stopProgress:
				return
			}
		}
	}()

	for range Keys.NumWorkers {
		go func() {
			defer wg.Done()
			for wi := range work {
				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
				if err != nil {
					if err == ErrNoNewArchiveData {
						atomic.AddInt32(&completed, 1)
						continue
					}
					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
					// Delete WAL immediately after successful snapshot.
					walPath := path.Join(wi.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL remove %s: %v", walPath, err)
					}
					successMu.Lock()
					successDirs = append(successDirs, wi.hostDir)
					successMu.Unlock()
				}
				atomic.AddInt32(&completed, 1)
			}
		}()
	}

	for i := range levels {
		hostDir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			hostDir:  hostDir,
			selector: selectors[i],
		}
	}
	close(work)
	wg.Wait()
	close(stopProgress)

	if errs > 0 {
		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
	}
	return int(n), successDirs, nil
}

// toCheckpointBinary writes a binary snapshot file for a single host-level node.
// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}
	if cf == nil {
		return ErrNoNewArchiveData
	}

	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
		return fmt.Errorf("mkdir %s: %w", dir, err)
	}

	// Write to a temp file first, then rename (atomic on POSIX).
	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))

	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
	}

	bw := bufio.NewWriter(f)
	if err := writeBinarySnapshotFile(bw, cf); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return fmt.Errorf("write binary snapshot: %w", err)
	}
	if err := bw.Flush(); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return err
	}
	f.Close()

	return os.Rename(tmpPath, finalPath)
}

// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
		return err
	}
	return writeBinaryLevel(w, cf)
}

// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
		return err
	}

	for name, metric := range cf.Metrics {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
			return err
		}
		for _, v := range metric.Data {
			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
				return err
			}
		}
	}

	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
		return err
	}

	for name, child := range cf.Children {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := writeBinaryLevel(w, child); err != nil {
			return err
		}
	}

	return nil
}

// writeString16 writes a 2-byte length-prefixed string to w.
func writeString16(w io.Writer, s string) error {
	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
		return err
	}
	_, err := io.WriteString(w, s)
	return err
}

// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
// The retention check (from) is applied to the file's 'to' timestamp.
func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	var magic uint32
	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
	}
	if magic != snapFileMagic {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
	}

	var fileFrom, fileTo int64
	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
	}
	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
	}

	if fileTo != 0 && fileTo < from {
		return nil // File is older than retention window, skip it
	}

	cf, err := readBinaryLevel(br)
	if err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
	}
	cf.From = fileFrom
	cf.To = fileTo

	return l.loadFile(cf, m)
}

// readBinaryLevel recursively reads a level from the binary snapshot format.
func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
	cf := &CheckpointFile{
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	var numMetrics uint32
	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
		return nil, fmt.Errorf("read num_metrics: %w", err)
	}

	for range numMetrics {
		name, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read metric name: %w", err)
		}

		var freq, start int64
		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
		}
		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
			return nil, fmt.Errorf("read start for %s: %w", name, err)
		}

		var numValues uint32
		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
		}

		data := make([]schema.Float, numValues)
		for i := range numValues {
			var bits uint32
			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
			}
			data[i] = schema.Float(math.Float32frombits(bits))
		}

		cf.Metrics[name] = &CheckpointMetrics{
			Frequency: freq,
			Start:     start,
			Data:      data,
		}
	}

	var numChildren uint32
	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
		return nil, fmt.Errorf("read num_children: %w", err)
	}

	for range numChildren {
		childName, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read child name: %w", err)
		}

		child, err := readBinaryLevel(r)
		if err != nil {
			return nil, fmt.Errorf("read child %s: %w", childName, err)
		}
		cf.Children[childName] = child
	}

	return cf, nil
}

// readString16 reads a 2-byte length-prefixed string from r.
func readString16(r io.Reader) (string, error) {
	var sLen uint16
	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
		return "", err
	}
	buf := make([]byte, sLen)
	if _, err := io.ReadFull(r, buf); err != nil {
		return "", err
	}
	return string(buf), nil
}


Let me check if there are other callers of `writeWALRecordDirect`:


pkg/metricstore/walCheckpoint.go:244:				if err := writeWALRecordDirect(ws.w, msg); err != nil {
pkg/metricstore/walCheckpoint.go:382:// writeWALRecordDirect encodes a WAL record into a contiguous buffer first,
pkg/metricstore/walCheckpoint.go:386:func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) (int, error) {


Good, only one caller. Let me also check line 441 — there may be an early return that still returns only error:


438
439		// Single atomic write to the buffered writer.
440		n, err := w.Write(buf)
441		return n, err
442	}
443
444	// readWALRecord reads one WAL record from the reader.
445	// Returns (nil, nil) on clean EOF. Returns error on data corruption.
446	// A CRC mismatch indicates a truncated trailing record (expected on crash).
447	func readWALRecord(r io.Reader) (*WALMessage, error) {
448		var magic uint32
449		if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
450			if err == io.EOF {
451				return nil, nil // Clean EOF
452			}

	// Single atomic write to the buffered writer.
	n, err := w.Write(buf)
	return n, err
}

// readWALRecord reads one WAL record from the reader.
// Returns (nil, nil) on clean EOF. Returns error on data corruption.
// A CRC mismatch indicates a truncated trailing record (expected on crash).
func readWALRecord(r io.Reader) (*WALMessage, error) {
	var magic uint32
	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
		if err == io.EOF {
			return nil, nil // Clean EOF
		}


The diagnostic at line 441 was stale — let me check if there's a `return err` somewhere in the function body I missed:


441:	return n, err
453:		return nil, fmt.Errorf("read record magic: %w", err)
462:		return nil, fmt.Errorf("read payload length: %w", err)
471:		return nil, fmt.Errorf("read payload: %w", err)
476:		return nil, fmt.Errorf("read CRC: %w", err)
560:		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
733:		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
743:		return err
750:		return fmt.Errorf("mkdir %s: %w", dir, err)
759:		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
766:		return fmt.Errorf("write binary snapshot: %w", err)
771:		return err
781:		return err
784:		return err
787:		return err
795:		return err
800:			return err
803:			return err
806:			return err
809:			return err
813:				return err
819:		return err
824:			return err
827:			return err
837:		return err
840:	return err
850:		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
858:		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
861:		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
870:		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)

[Showing results with pagination = limit: 30]


OK, that diagnostic was likely stale from before my edit. Now update the caller in `processMsg`:


Edit

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}
				if err := writeWALRecordDirect(ws.w, msg); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
				ws.dirty = true
			}

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}

				// Enforce max WAL size: force-rotate before writing if limit is exceeded.
				// The in-memory store still holds the data; only crash-recovery coverage is lost.
				if maxSize := Keys.Checkpoints.MaxWALSize; maxSize > 0 && ws.size >= maxSize {
					cclog.Warnf("[METRICSTORE]> WAL: force-rotating %s (size %d >= limit %d)",
						hostDir, ws.size, maxSize)
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, hostDir)
					ws = getOrOpenWAL(hostDir)
					if ws == nil {
						return
					}
				}

				n, err := writeWALRecordDirect(ws.w, msg)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
				ws.size += int64(n)
				ws.dirty = true
			}


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go has been updated successfully.

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}

				// Enforce max WAL size: force-rotate before writing if limit is exceeded.
				// The in-memory store still holds the data; only crash-recovery coverage is lost.
				if maxSize := Keys.Checkpoints.MaxWALSize; maxSize > 0 && ws.size >= maxSize {
					cclog.Warnf("[METRICSTORE]> WAL: force-rotating %s (size %d >= limit %d)",
						hostDir, ws.size, maxSize)
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, hostDir)
					ws = getOrOpenWAL(hostDir)
					if ws == nil {
						return
					}
				}

				n, err := writeWALRecordDirect(ws.w, msg)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
				ws.size += int64(n)
				ws.dirty = true
			}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼
//	        WALStaging goroutine
//	               │
//	               ▼
//	        checkpoints/cluster/host/current.wal  (append-only, binary)
//
//	Periodic checkpoint (Checkpointing goroutine):
//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
//	  2. Signal WALStaging to truncate current.wal per host
//
//	On restart (FromCheckpoint):
//	  1. Load most recent <timestamp>.bin snapshot
//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
//
// # WAL Record Format
//
//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
//
//	payload:
//	  [8B timestamp int64]
//	  [2B metric_name_len uint16][N metric name bytes]
//	  [1B selector_count uint8]
//	  per selector: [1B selector_len uint8][M selector bytes]
//	  [4B value float32 bits]
//
// # Binary Snapshot Format
//
//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
//	Level tree (recursive):
//	  [4B num_metrics uint32]
//	  per metric:
//	    [2B name_len uint16][N name bytes]
//	    [8B frequency int64][8B start int64]
//	    [4B num_values uint32][num_values × 4B float32]
//	  [4B num_children uint32]
//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
package metricstore

import (
	"bufio"
	"context"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"hash/fnv"
	"io"
	"math"
	"os"
	"path"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Magic numbers for binary formats.
const (
	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
)

// walShardChs holds per-shard channels for WAL messages.
// Initialized by WALStaging; nil when WAL is not active.
var walShardChs []chan *WALMessage

// walShardRotateChs holds per-shard channels for WAL rotation requests.
var walShardRotateChs []chan walRotateReq

// walNumShards stores the number of shards (set during WALStaging init).
var walNumShards int

// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
var walStagingWg sync.WaitGroup

// walShuttingDown is set before closing shard channels to prevent
// SendWALMessage from sending on a closed channel (which panics in Go).
var walShuttingDown atomic.Bool

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f     *os.File
	w     *bufio.Writer
	dirty bool
	size  int64 // approximate bytes written (tracked from open + writes)
}

// walFlushInterval controls how often dirty WAL files are flushed to disk.
// Decoupling flushes from message processing lets the consumer run at memory
// speed, amortizing syscall overhead across many writes.
const walFlushInterval = 1 * time.Second

// walShardIndex computes which shard a message belongs to based on cluster+node.
// Uses FNV-1a hash for fast, well-distributed mapping.
func walShardIndex(cluster, node string) int {
	h := fnv.New32a()
	h.Write([]byte(cluster))
	h.Write([]byte{0})
	h.Write([]byte(node))
	return int(h.Sum32()) % walNumShards
}

// SendWALMessage routes a WAL message to the appropriate shard channel.
// Returns false if the channel is full or shutdown is in progress.
func SendWALMessage(msg *WALMessage) bool {
	if walShardChs == nil || walShuttingDown.Load() {
		return false
	}
	shard := walShardIndex(msg.Cluster, msg.Node)
	select {
	case walShardChs[shard] <- msg:
		return true
	default:
		return false
	}
}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.
// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return
	}

	walNumShards = max(Keys.NumWorkers, 1)
	chBufSize := max(65536/walNumShards, 1024)

	walShardChs = make([]chan *WALMessage, walNumShards)
	walShardRotateChs = make([]chan walRotateReq, walNumShards)

	for i := range walNumShards {
		walShardChs[i] = make(chan *WALMessage, chBufSize)
		walShardRotateChs[i] = make(chan walRotateReq, 64)
	}

	for i := range walNumShards {
		msgCh := walShardChs[i]
		rotateCh := walShardRotateChs[i]

		walStagingWg.Add(1)
		wg.Go(func() {
			defer walStagingWg.Done()
			hostFiles := make(map[string]*walFileState)

			defer func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
						ws.f.Close()
					}
				}
			}()

			getOrOpenWAL := func(hostDir string) *walFileState {
				ws, ok := hostFiles[hostDir]
				if ok {
					return ws
				}

				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
					return nil
				}

				walPath := path.Join(hostDir, "current.wal")
				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
					return nil
				}

				w := bufio.NewWriter(f)

				// Write file header magic if file is new (empty).
				info, err := f.Stat()
				var fileSize int64
				if err == nil {
					fileSize = info.Size()
				}
				if err == nil && fileSize == 0 {
					var hdr [4]byte
					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
					if _, err := w.Write(hdr[:]); err != nil {
						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
						f.Close()
						return nil
					}
					fileSize = 4
				}

				ws = &walFileState{f: f, w: w, size: fileSize}
				hostFiles[hostDir] = ws
				return ws
			}

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}
				if err := writeWALRecordDirect(ws.w, msg); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
				ws.dirty = true
			}

			processRotate := func(req walRotateReq) {
				ws, ok := hostFiles[req.hostDir]
				if ok && ws.f != nil {
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(req.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, req.hostDir)
				}
				close(req.done)
			}

			flushDirty := func() {
				for _, ws := range hostFiles {
					if ws.dirty {
						ws.w.Flush()
						ws.dirty = false
					}
				}
			}

			ticker := time.NewTicker(walFlushInterval)
			defer ticker.Stop()

			// drainBatch processes up to 4096 pending messages without blocking.
			// Returns false if the channel was closed.
			drainBatch := func() bool {
				for range 4096 {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							flushDirty()
							return false
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						return true
					}
				}
				return true
			}

			for {
				select {
				case <-ctx.Done():
					// On shutdown, skip draining buffered messages — a full binary
					// checkpoint will be written from in-memory state, making
					// buffered WAL records redundant.
					flushDirty()
					return
				case msg, ok := <-msgCh:
					if !ok {
						return
					}
					processMsg(msg)
					if !drainBatch() {
						return
					}
					// No flush here — timer handles periodic flushing.
				case <-ticker.C:
					flushDirty()
				case req := <-rotateCh:
					processRotate(req)
				}
			}
		})
	}
}

// WaitForWALStagingDrain blocks until all WALStaging goroutines have exited.
// Must be called after closing walShardChs to ensure all file handles are
// flushed and closed before checkpoint writes begin.
func WaitForWALStagingDrain() {
	walStagingWg.Wait()
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete. Each request is routed to the
// shard that owns the host directory.
//
// If shutdown is in progress (WAL staging goroutines may have exited),
// rotation is skipped to avoid deadlocking on abandoned channels.
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		default:
			// Channel full or goroutine not consuming — skip this host.
			cclog.Warnf("[METRICSTORE]> WAL rotation skipped for %s (channel full)", dir)
		}
	}
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation timed out, continuing")
			return
		}
	}
}

// walShardIndexFromDir extracts cluster and node from a host directory path
// and returns the shard index. The path format is: rootDir/cluster/node
func walShardIndexFromDir(hostDir string) int {
	// hostDir = .../cluster/node — extract last two path components
	node := path.Base(hostDir)
	cluster := path.Base(path.Dir(hostDir))
	return walShardIndex(cluster, node)
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete.
func RotateWALFilesAfterShutdown(hostDirs []string) {
	for _, dir := range hostDirs {
		walPath := path.Join(dir, "current.wal")
		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
		}
	}
}

// writeWALRecordDirect encodes a WAL record into a contiguous buffer first,
// then writes it to the bufio.Writer in a single call. This prevents partial
// records in the write buffer if a write error occurs mid-record (e.g. disk full).
// Returns the number of bytes written and any error.
func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) (int, error) {
	// Compute payload and total record size.
	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
	for _, s := range msg.Selector {
		payloadSize += 1 + len(s)
	}
	// Total: 8 (header) + payload + 4 (CRC).
	totalSize := 8 + payloadSize + 4

	// Use stack buffer for typical small records, heap-allocate only for large ones.
	var stackBuf [256]byte
	var buf []byte
	if totalSize <= len(stackBuf) {
		buf = stackBuf[:totalSize]
	} else {
		buf = make([]byte, totalSize)
	}

	// Header: magic + payload length.
	binary.LittleEndian.PutUint32(buf[0:4], walRecordMagic)
	binary.LittleEndian.PutUint32(buf[4:8], uint32(payloadSize))

	// Payload starts at offset 8.
	p := 8

	// Timestamp (8 bytes).
	binary.LittleEndian.PutUint64(buf[p:p+8], uint64(msg.Timestamp))
	p += 8

	// Metric name length (2 bytes) + metric name.
	binary.LittleEndian.PutUint16(buf[p:p+2], uint16(len(msg.MetricName)))
	p += 2
	p += copy(buf[p:], msg.MetricName)

	// Selector count (1 byte).
	buf[p] = byte(len(msg.Selector))
	p++

	// Selectors (1-byte length + bytes each).
	for _, sel := range msg.Selector {
		buf[p] = byte(len(sel))
		p++
		p += copy(buf[p:], sel)
	}

	// Value (4 bytes, float32 bits).
	binary.LittleEndian.PutUint32(buf[p:p+4], math.Float32bits(float32(msg.Value)))
	p += 4

	// CRC32 over payload (bytes 8..8+payloadSize).
	crc := crc32.ChecksumIEEE(buf[8 : 8+payloadSize])
	binary.LittleEndian.PutUint32(buf[p:p+4], crc)

	// Single atomic write to the buffered writer.
	n, err := w.Write(buf)
	return n, err
}

// readWALRecord reads one WAL record from the reader.
// Returns (nil, nil) on clean EOF. Returns error on data corruption.
// A CRC mismatch indicates a truncated trailing record (expected on crash).
func readWALRecord(r io.Reader) (*WALMessage, error) {
	var magic uint32
	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
		if err == io.EOF {
			return nil, nil // Clean EOF
		}
		return nil, fmt.Errorf("read record magic: %w", err)
	}

	if magic != walRecordMagic {
		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
	}

	var payloadLen uint32
	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
		return nil, fmt.Errorf("read payload length: %w", err)
	}

	if payloadLen > 1<<20 { // 1 MB sanity limit
		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
	}

	payload := make([]byte, payloadLen)
	if _, err := io.ReadFull(r, payload); err != nil {
		return nil, fmt.Errorf("read payload: %w", err)
	}

	var storedCRC uint32
	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
		return nil, fmt.Errorf("read CRC: %w", err)
	}

	if crc32.ChecksumIEEE(payload) != storedCRC {
		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
	}

	return parseWALPayload(payload)
}

// parseWALPayload decodes a binary payload into a WALMessage.
func parseWALPayload(payload []byte) (*WALMessage, error) {
	if len(payload) < 8+2+1+4 {
		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
	}

	offset := 0

	// Timestamp (8 bytes)
	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
	offset += 8

	// Metric name (2-byte length + bytes)
	if offset+2 > len(payload) {
		return nil, fmt.Errorf("metric name length overflows payload")
	}
	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
	offset += 2

	if offset+mLen > len(payload) {
		return nil, fmt.Errorf("metric name overflows payload")
	}
	metricName := string(payload[offset : offset+mLen])
	offset += mLen

	// Selector count (1 byte)
	if offset >= len(payload) {
		return nil, fmt.Errorf("selector count overflows payload")
	}
	selCount := int(payload[offset])
	offset++

	selectors := make([]string, selCount)
	for i := range selCount {
		if offset >= len(payload) {
			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
		}
		sLen := int(payload[offset])
		offset++

		if offset+sLen > len(payload) {
			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
		}
		selectors[i] = string(payload[offset : offset+sLen])
		offset += sLen
	}

	// Value (4 bytes, float32 bits)
	if offset+4 > len(payload) {
		return nil, fmt.Errorf("value overflows payload")
	}
	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
	value := schema.Float(math.Float32frombits(bits))

	return &WALMessage{
		MetricName: metricName,
		Timestamp:  ts,
		Selector:   selectors,
		Value:      value,
	}, nil
}

// loadWALFile reads a WAL file and replays all valid records into the Level tree.
// l is the host-level node. Corrupt or partial trailing records are silently skipped
// (expected on crash). Records older than 'from' are skipped.
func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	// Verify file header magic.
	var fileMagic uint32
	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
		if err == io.EOF {
			return nil // Empty file, no data
		}
		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
	}

	if fileMagic != walFileMagic {
		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
	}

	// Cache level lookups to avoid repeated tree traversal.
	lvlCache := make(map[string]*Level)

	for {
		msg, err := readWALRecord(br)
		if err != nil {
			// Truncated trailing record is expected after a crash; stop replaying.
			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
			break
		}
		if msg == nil {
			break // Clean EOF
		}

		if msg.Timestamp < from {
			continue // Older than retention window
		}

		minfo, ok := m.Metrics[msg.MetricName]
		if !ok {
			continue // Unknown metric (config may have changed)
		}

		// Cache key is the null-separated selector path.
		cacheKey := joinSelector(msg.Selector)
		lvl, ok := lvlCache[cacheKey]
		if !ok {
			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
			lvlCache[cacheKey] = lvl
		}

		// Write directly to the buffer, same as WriteToLevel but without the
		// global level lookup (we already have the right level).
		lvl.lock.Lock()
		b := lvl.metrics[minfo.offset]
		if b == nil {
			b = newBuffer(msg.Timestamp, minfo.Frequency)
			lvl.metrics[minfo.offset] = b
		}
		nb, writeErr := b.write(msg.Timestamp, msg.Value)
		if writeErr == nil && b != nb {
			lvl.metrics[minfo.offset] = nb
		}
		// Ignore write errors for timestamps before buffer start (can happen when
		// replaying WAL entries that predate a loaded snapshot's start time).
		lvl.lock.Unlock()
	}

	return nil
}

// joinSelector builds a cache key from a selector slice using null bytes as separators.
func joinSelector(sel []string) string {
	if len(sel) == 0 {
		return ""
	}
	var result strings.Builder
	result.WriteString(sel[0])
	for i := 1; i < len(sel); i++ {
		result.WriteString("\x00" + sel[i])
	}
	return result.String()
}

// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
// Returns the number of files written, the list of host directories that were
// successfully checkpointed (for WAL rotation), and any errors.
func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
	// Collect all cluster/host pairs.
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		hostDir  string
		selector []string
	}

	totalWork := len(levels)
	cclog.Infof("[METRICSTORE]> Starting binary checkpoint for %d hosts with %d workers", totalWork, Keys.NumWorkers)

	n, errs, completed := int32(0), int32(0), int32(0)
	var successDirs []string
	var successMu sync.Mutex

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)

	// Progress logging goroutine.
	stopProgress := make(chan struct{})
	go func() {
		ticker := time.NewTicker(10 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				cclog.Infof("[METRICSTORE]> Checkpoint progress: %d/%d hosts (%d written, %d errors)",
					atomic.LoadInt32(&completed), totalWork, atomic.LoadInt32(&n), atomic.LoadInt32(&errs))
			case <-stopProgress:
				return
			}
		}
	}()

	for range Keys.NumWorkers {
		go func() {
			defer wg.Done()
			for wi := range work {
				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
				if err != nil {
					if err == ErrNoNewArchiveData {
						atomic.AddInt32(&completed, 1)
						continue
					}
					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
					// Delete WAL immediately after successful snapshot.
					walPath := path.Join(wi.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL remove %s: %v", walPath, err)
					}
					successMu.Lock()
					successDirs = append(successDirs, wi.hostDir)
					successMu.Unlock()
				}
				atomic.AddInt32(&completed, 1)
			}
		}()
	}

	for i := range levels {
		hostDir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			hostDir:  hostDir,
			selector: selectors[i],
		}
	}
	close(work)
	wg.Wait()
	close(stopProgress)

	if errs > 0 {
		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
	}
	return int(n), successDirs, nil
}

// toCheckpointBinary writes a binary snapshot file for a single host-level node.
// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}
	if cf == nil {
		return ErrNoNewArchiveData
	}

	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
		return fmt.Errorf("mkdir %s: %w", dir, err)
	}

	// Write to a temp file first, then rename (atomic on POSIX).
	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))

	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
	}

	bw := bufio.NewWriter(f)
	if err := writeBinarySnapshotFile(bw, cf); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return fmt.Errorf("write binary snapshot: %w", err)
	}
	if err := bw.Flush(); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return err
	}
	f.Close()

	return os.Rename(tmpPath, finalPath)
}

// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
		return err
	}
	return writeBinaryLevel(w, cf)
}

// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
		return err
	}

	for name, metric := range cf.Metrics {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
			return err
		}
		for _, v := range metric.Data {
			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
				return err
			}
		}
	}

	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
		return err
	}

	for name, child := range cf.Children {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := writeBinaryLevel(w, child); err != nil {
			return err
		}
	}

	return nil
}

// writeString16 writes a 2-byte length-prefixed string to w.
func writeString16(w io.Writer, s string) error {
	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
		return err
	}
	_, err := io.WriteString(w, s)
	return err
}

// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
// The retention check (from) is applied to the file's 'to' timestamp.
func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	var magic uint32
	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
	}
	if magic != snapFileMagic {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
	}

	var fileFrom, fileTo int64
	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
	}
	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
	}

	if fileTo != 0 && fileTo < from {
		return nil // File is older than retention window, skip it
	}

	cf, err := readBinaryLevel(br)
	if err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
	}
	cf.From = fileFrom
	cf.To = fileTo

	return l.loadFile(cf, m)
}

// readBinaryLevel recursively reads a level from the binary snapshot format.
func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
	cf := &CheckpointFile{
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	var numMetrics uint32
	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
		return nil, fmt.Errorf("read num_metrics: %w", err)
	}

	for range numMetrics {
		name, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read metric name: %w", err)
		}

		var freq, start int64
		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
		}
		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
			return nil, fmt.Errorf("read start for %s: %w", name, err)
		}

		var numValues uint32
		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
		}

		data := make([]schema.Float, numValues)
		for i := range numValues {
			var bits uint32
			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
			}
			data[i] = schema.Float(math.Float32frombits(bits))
		}

		cf.Metrics[name] = &CheckpointMetrics{
			Frequency: freq,
			Start:     start,
			Data:      data,
		}
	}

	var numChildren uint32
	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
		return nil, fmt.Errorf("read num_children: %w", err)
	}

	for range numChildren {
		childName, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read child name: %w", err)
		}

		child, err := readBinaryLevel(r)
		if err != nil {
			return nil, fmt.Errorf("read child %s: %w", childName, err)
		}
		cf.Children[childName] = child
	}

	return cf, nil
}

// readString16 reads a 2-byte length-prefixed string from r.
func readString16(r io.Reader) (string, error) {
	var sLen uint16
	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
		return "", err
	}
	buf := make([]byte, sLen)
	if _, err := io.ReadFull(r, buf); err != nil {
		return "", err
	}
	return string(buf), nil
}

-        if err := writeWALRecordDirect(ws.w, msg); err != nil {

+        // Enforce max WAL size: force-rotate before writing if limit is exceeded.

+        // The in-memory store still holds the data; only crash-recovery coverage is lost.

+        if maxSize := Keys.Checkpoints.MaxWALSize; maxSize > 0 && ws.size >= maxSize {

+          cclog.Warnf("[METRICSTORE]> WAL: force-rotating %s (size %d >= limit %d)",

+          walPath := path.Join(hostDir, "current.wal")

+          if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {

+            cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)

+        n, err := writeWALRecordDirect(ws.w, msg)


Updated task #3 status


Updated task #4 status


Bash

Build all packages


(Bash completed with no output)


Run metricstore tests


ok  	github.com/ClusterCockpit/cc-backend/pkg/metricstore	0.429s


Build and tests pass.


All done. Here's a summary of the changes:

**`pkg/metricstore/checkpoint.go`** — Fixed partial checkpoint success handling. Previously, if `ToCheckpointWAL` returned any error (even with some hosts succeeding), `lastCheckpoint` was never advanced and `RotateWALFiles` was never called. Now, if `n > 0` snapshots were written, the timestamp advances and WAL rotation runs for the successful hosts, independent of whether other hosts failed.

**`pkg/metricstore/config.go`** — Added `MaxWALSize int64` field to `Checkpoints` struct. Configurable via `"max-wal-size"` in JSON (bytes). Default 0 = unlimited (backward compatible).

**`pkg/metricstore/walCheckpoint.go`** — Three changes:
1. Added `size int64` field to `walFileState` to track approximate file size
2. Initialized size from `Stat()` in `getOrOpenWAL()` (reusing the existing stat call)
3. Added size enforcement in `processMsg()`: when a host's WAL exceeds `MaxWALSize`, it's force-rotated (flush, close, delete, re-open). This accepts loss of crash-recovery coverage for that interval, but the in-memory store still holds the data for the next successful checkpoint.
4. Changed `writeWALRecordDirect` to return `(int, error)` so the caller can track bytes written.


Update documentation and config schema accordingly


Agent

Find checkpoint config docs/schema

Prompt:
Find all files that document the checkpoint configuration, specifically the "checkpoints" JSON config block with "file-format" and "directory" fields. Look for:

1. Any JSON schema files (*.json with "$schema" or "properties" mentioning "checkpoints")
2. Documentation files mentioning checkpoint configuration (e.g. in docs/, configs/, README)
3. Example config files (config.json, *.example.json, etc.)
4. CLAUDE.md or any markdown that documents the checkpoints config structure
5. The config.go file itself (already known: pkg/metricstore/config.go)

Search broadly — glob for *.md, *.json, *.yml, *.yaml in the project root and key subdirectories. Use grep for "max-wal-size", "file-format", "checkpoints" in config-related contexts.


Excellent! Now I have all the information needed. Let me compile the comprehensive report.

## Summary: Checkpoint Configuration Documentation

I found comprehensive documentation about the checkpoint configuration scattered across multiple files in the project. Here's what documents the "checkpoints" JSON config block with "file-format" and "directory" fields:

### Primary Documentation Files

**1. Configuration Schema (Most authoritative):**
- **File**: `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/configSchema.go`
  - Contains embedded JSON schema defining the checkpoints block structure
  - Defines `file-format` field: "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'wal'."
  - Defines `directory` field: "Path in which the checkpointed files should be placed."
  - Also documents `max-wal-size` field (not visible in basic schema but referenced in code)

**2. Configuration Structure Documentation (Go Code):**
- **File**: `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go` (lines 59-70)
  - Defines the `Checkpoints` struct with detailed docstring comments
  - Documents fields:
    - `FileFormat`: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal"
    - `RootDir`: Filesystem path for checkpoint files (created if missing)
    - `MaxWALSize`: Maximum size in bytes for a single host's WAL file; 0 = unlimited (default)

**3. Checkpoint Implementation Documentation:**
- **File**: `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go` (lines 6-34)
  - Documents file organization and checkpoint workflow
  - Describes file structure:
    ```
    checkpoints/
      cluster1/
        host001/
          1234567890.json  (JSON format: full subtree snapshot)
          1234567890.bin   (binary format: full subtree snapshot)
          current.wal      (WAL format: append-only per-entry log)
    ```

**4. WAL Implementation Details:**
- **File**: `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go` (lines 6-50)
  - Documents WAL record format and binary snapshot format
  - Explains the crash-safety mechanism

### Example Configuration Files

**5. Example Configuration with Checkpoints:**
- **File**: `/Users/jan/prg/ClusterCockpit/cc-backend/configs/config.json` (lines 76-78)
  - Example:
    ```json
    "metric-store": {
      "checkpoints": {
        "directory": "./var/checkpoints"
      },
      ...
    }
    ```

**6. Minimal Demo Configuration:**
- **File**: `/Users/jan/prg/ClusterCockpit/cc-backend/configs/config-demo.json`
  - Minimal example (uses defaults for checkpoint fields)

### Developer Guidance

**7. CLAUDE.md (Developer Guidelines):**
- **File**: `/Users/jan/prg/ClusterCockpit/cc-backend/CLAUDE.md` (lines 99-101)
  - Brief mention: "In-memory metric storage with checkpointing"

**8. Release Notes:**
- **File**: `/Users/jan/prg/ClusterCockpit/cc-backend/ReleaseNotes.md`
  - Documents checkpoint archiving changes and cleanup features
  - References the `-cleanup-checkpoints` CLI flag

**9. CLI Documentation:**
- **File**: `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/cli.go` (line 32)
  - Flag: `-cleanup-checkpoints`: "Clean up old checkpoint files (delete or archive) based on retention settings, then exit"

### README Documentation

**10. Main README:**
- **File**: `/Users/jan/prg/ClusterCockpit/cc-backend/README.md`
  - General project overview (no specific checkpoint config details)

### Configuration Hierarchy

The checkpoint configuration is nested as:
```json
{
  "metric-store": {
    "checkpoints": {
      "file-format": "wal|json",
      "directory": "/path/to/checkpoints",
      "max-wal-size": 0
    }
  }
}
```

All files are located in the `/Users/jan/prg/ClusterCockpit/cc-backend` repository. The most authoritative sources are the config schema and the config.go Go file with its docstrings.

Prompt:
Find all files that document the checkpoint configuration, specifically the "checkpoints" JSON config block with "file-format" and "directory" fields. Look for:

1. Any JSON schema files (*.json with "$schema" or "properties" mentioning "checkpoints")
2. Documentation files mentioning checkpoint configuration (e.g. in docs/, configs/, README)
3. Example config files (config.json, *.example.json, etc.)
4. CLAUDE.md or any markdown that documents the checkpoints config structure
5. The config.go file itself (already known: pkg/metricstore/config.go)

Search broadly — glob for *.md, *.json, *.yml, *.yaml in the project root and key subdirectories. Use grep for "max-wal-size", "file-format", "checkpoints" in config-related contexts.


1	// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
2	// All rights reserved. This file is part of cc-backend.
3	// Use of this source code is governed by a MIT-style
4	// license that can be found in the LICENSE file.
5
6	package metricstore
7
8	const configSchema = `{
9	  "type": "object",
10	  "description": "Configuration specific to built-in metric-store.",
11	  "properties": {
12	    "num-workers": {
13	      "description": "Number of concurrent workers for checkpoint and archive operations",
14	      "type": "integer"
15	    },
16	    "checkpoints": {
17	      "description": "Configuration for checkpointing the metrics buffers",
18	      "type": "object",
19	      "properties": {
20	        "file-format": {
21	          "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'wal'.",
22	          "type": "string"
23	        },
24	        "directory": {
25	          "description": "Path in which the checkpointed files should be placed.",
26	          "type": "string"
27	        }
28	      }
29	    },
30	    "cleanup": {
31	      "description": "Configuration for the cleanup process. The cleanup interval is always 'retention-in-memory'.",
32	      "type": "object",
33	      "properties": {
34	        "mode": {
35	          "description": "The operation mode (e.g., 'archive' or 'delete').",
36	          "type": "string",
37	          "enum": ["archive", "delete"]
38	        },
39	        "directory": {
40	          "description": "Target directory for archive operations.",
41	          "type": "string"
42	        }
43	      },
44	      "if": {
45	        "properties": {
46	          "mode": { "const": "archive" }
47	        }
48	      },
49	      "then": {
50	        "required": ["directory"]
51	      }
52	    },
53	    "retention-in-memory": {
54	      "description": "Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.",
55	      "type": "string"
56	    },
57	    "checkpoint-interval": {
58	      "description": "Interval between checkpoints as a Go duration string (e.g., '12h', '6h', '30m'). Default is '12h'.",
59	      "type": "string"
60	    },
61	    "memory-cap": {
62	      "description": "Upper memory capacity limit used by metricstore in GB",
63	      "type": "integer"
64	    },
65	    "nats-subscriptions": {
66	      "description": "Array of various subscriptions. Allows to subscribe to different subjects and publishers.",
67	      "type": "array",
68	      "items": {
69	        "type": "object",
70	        "properties": {
71	          "subscribe-to": {
72	            "description": "Subject name",
73	            "type": "string"
74	          },
75	          "cluster-tag": {
76	            "description": "Optional: Allow lines without a cluster tag, use this as default",
77	            "type": "string"
78	          }
79	        },
80					"required": ["subscribe-to"]
81	      }
82	    }
83	  },
84	  "required": ["retention-in-memory", "memory-cap"]
85	}`
86

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

const configSchema = `{
  "type": "object",
  "description": "Configuration specific to built-in metric-store.",
  "properties": {
    "num-workers": {
      "description": "Number of concurrent workers for checkpoint and archive operations",
      "type": "integer"
    },
    "checkpoints": {
      "description": "Configuration for checkpointing the metrics buffers",
      "type": "object",
      "properties": {
        "file-format": {
          "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'wal'.",
          "type": "string"
        },
        "directory": {
          "description": "Path in which the checkpointed files should be placed.",
          "type": "string"
        }
      }
    },
    "cleanup": {
      "description": "Configuration for the cleanup process. The cleanup interval is always 'retention-in-memory'.",
      "type": "object",
      "properties": {
        "mode": {
          "description": "The operation mode (e.g., 'archive' or 'delete').",
          "type": "string",
          "enum": ["archive", "delete"]
        },
        "directory": {
          "description": "Target directory for archive operations.",
          "type": "string"
        }
      },
      "if": {
        "properties": {
          "mode": { "const": "archive" }
        }
      },
      "then": {
        "required": ["directory"]
      }
    },
    "retention-in-memory": {
      "description": "Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.",
      "type": "string"
    },
    "checkpoint-interval": {
      "description": "Interval between checkpoints as a Go duration string (e.g., '12h', '6h', '30m'). Default is '12h'.",
      "type": "string"
    },
    "memory-cap": {
      "description": "Upper memory capacity limit used by metricstore in GB",
      "type": "integer"
    },
    "nats-subscriptions": {
      "description": "Array of various subscriptions. Allows to subscribe to different subjects and publishers.",
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "subscribe-to": {
            "description": "Subject name",
            "type": "string"
          },
          "cluster-tag": {
            "description": "Optional: Allow lines without a cluster tag, use this as default",
            "type": "string"
          }
        },
				"required": ["subscribe-to"]
      }
    }
  },
  "required": ["retention-in-memory", "memory-cap"]
}`


70	      "scope": "alex",
71	      "url": "http://x.x.x.x:8084",
72	      "token": "MySecret"
73	    }
74	  ],
75	  "metric-store": {
76	    "checkpoints": {
77	      "directory": "./var/checkpoints"
78	    },
79	    "memory-cap": 100,
80	    "retention-in-memory": "24h",
81	    "cleanup": {
82	      "mode": "archive",
83	      "directory": "./var/archive"
84	    },
85	    "nats-subscriptions": [
86	      {
87	        "subscribe-to": "hpc-nats",
88	        "cluster-tag": "fritz"
89	      },

      "scope": "alex",
      "url": "http://x.x.x.x:8084",
      "token": "MySecret"
    }
  ],
  "metric-store": {
    "checkpoints": {
      "directory": "./var/checkpoints"
    },
    "memory-cap": 100,
    "retention-in-memory": "24h",
    "cleanup": {
      "mode": "archive",
      "directory": "./var/archive"
    },
    "nats-subscriptions": [
      {
        "subscribe-to": "hpc-nats",
        "cluster-tag": "fritz"
      },


Now I'll update the config schema and the example config.


Edit

        "directory": {
          "description": "Path in which the checkpointed files should be placed.",
          "type": "string"
        }
      }

        "directory": {
          "description": "Path in which the checkpointed files should be placed.",
          "type": "string"
        },
        "max-wal-size": {
          "description": "Maximum size in bytes for a single host's WAL file. When exceeded the WAL is force-rotated to prevent unbounded disk growth. Only applies when file-format is 'wal'. 0 means unlimited (default).",
          "type": "integer",
          "minimum": 0
        }
      }


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/configSchema.go has been updated successfully.

        "directory": {
          "description": "Path in which the checkpointed files should be placed.",
          "type": "string"
        },
        "max-wal-size": {
          "description": "Maximum size in bytes for a single host's WAL file. When exceeded the WAL is force-rotated to prevent unbounded disk growth. Only applies when file-format is 'wal'. 0 means unlimited (default).",
          "type": "integer",
          "minimum": 0
        }
      }

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

const configSchema = `{
  "type": "object",
  "description": "Configuration specific to built-in metric-store.",
  "properties": {
    "num-workers": {
      "description": "Number of concurrent workers for checkpoint and archive operations",
      "type": "integer"
    },
    "checkpoints": {
      "description": "Configuration for checkpointing the metrics buffers",
      "type": "object",
      "properties": {
        "file-format": {
          "description": "Specify the format for checkpoint files. Two variants: 'json' (human-readable, periodic) and 'wal' (binary snapshot + Write-Ahead Log, crash-safe). Default is 'wal'.",
          "type": "string"
        },
        "directory": {
          "description": "Path in which the checkpointed files should be placed.",
          "type": "string"
        }
      }
    },
    "cleanup": {
      "description": "Configuration for the cleanup process. The cleanup interval is always 'retention-in-memory'.",
      "type": "object",
      "properties": {
        "mode": {
          "description": "The operation mode (e.g., 'archive' or 'delete').",
          "type": "string",
          "enum": ["archive", "delete"]
        },
        "directory": {
          "description": "Target directory for archive operations.",
          "type": "string"
        }
      },
      "if": {
        "properties": {
          "mode": { "const": "archive" }
        }
      },
      "then": {
        "required": ["directory"]
      }
    },
    "retention-in-memory": {
      "description": "Keep the metrics within memory for given time interval. Retention for X hours, then the metrics would be freed.",
      "type": "string"
    },
    "checkpoint-interval": {
      "description": "Interval between checkpoints as a Go duration string (e.g., '12h', '6h', '30m'). Default is '12h'.",
      "type": "string"
    },
    "memory-cap": {
      "description": "Upper memory capacity limit used by metricstore in GB",
      "type": "integer"
    },
    "nats-subscriptions": {
      "description": "Array of various subscriptions. Allows to subscribe to different subjects and publishers.",
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "subscribe-to": {
            "description": "Subject name",
            "type": "string"
          },
          "cluster-tag": {
            "description": "Optional: Allow lines without a cluster tag, use this as default",
            "type": "string"
          }
        },
				"required": ["subscribe-to"]
      }
    }
  },
  "required": ["retention-in-memory", "memory-cap"]
}`

+          "description": "Maximum size in bytes for a single host's WAL file. When exceeded the WAL is force-rotated to prevent unbounded disk growth. Only applies when file-format is 'wal'. 0 means unlimited (default).",


Now let me check if there's additional documentation in walCheckpoint.go's header that should mention this:


1	// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
2	// All rights reserved. This file is part of cc-backend.
3	// Use of this source code is governed by a MIT-style
4	// license that can be found in the LICENSE file.
5
6	// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
7	//
8	// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
9	// per host, eliminating the extra memory overhead of the AvroStore and providing
10	// truly continuous (per-write) crash safety.
11	//
12	// # Architecture
13	//
14	//	Metric write (DecodeLine)
15	//	      │
16	//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
17	//	      │
18	//	      └─► WALMessages channel
19	//	               │
20	//	               ▼
21	//	        WALStaging goroutine
22	//	               │
23	//	               ▼
24	//	        checkpoints/cluster/host/current.wal  (append-only, binary)
25	//
26	//	Periodic checkpoint (Checkpointing goroutine):
27	//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
28	//	  2. Signal WALStaging to truncate current.wal per host
29	//
30	//	On restart (FromCheckpoint):
31	//	  1. Load most recent <timestamp>.bin snapshot
32	//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
33	//
34	// # WAL Record Format
35	//
36	//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
37	//
38	//	payload:
39	//	  [8B timestamp int64]
40	//	  [2B metric_name_len uint16][N metric name bytes]
41	//	  [1B selector_count uint8]
42	//	  per selector: [1B selector_len uint8][M selector bytes]
43	//	  [4B value float32 bits]
44	//
45	// # Binary Snapshot Format
46	//
47	//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
48	//	Level tree (recursive):
49	//	  [4B num_metrics uint32]
50	//	  per metric:
51	//	    [2B name_len uint16][N name bytes]
52	//	    [8B frequency int64][8B start int64]
53	//	    [4B num_values uint32][num_values × 4B float32]
54	//	  [4B num_children uint32]
55	//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼
//	        WALStaging goroutine
//	               │
//	               ▼
//	        checkpoints/cluster/host/current.wal  (append-only, binary)
//
//	Periodic checkpoint (Checkpointing goroutine):
//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
//	  2. Signal WALStaging to truncate current.wal per host
//
//	On restart (FromCheckpoint):
//	  1. Load most recent <timestamp>.bin snapshot
//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
//
// # WAL Record Format
//
//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
//
//	payload:
//	  [8B timestamp int64]
//	  [2B metric_name_len uint16][N metric name bytes]
//	  [1B selector_count uint8]
//	  per selector: [1B selector_len uint8][M selector bytes]
//	  [4B value float32 bits]
//
// # Binary Snapshot Format
//
//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
//	Level tree (recursive):
//	  [4B num_metrics uint32]
//	  per metric:
//	    [2B name_len uint16][N name bytes]
//	    [8B frequency int64][8B start int64]
//	    [4B num_values uint32][num_values × 4B float32]
//	  [4B num_children uint32]
//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)


The header is about the internal architecture — no need to add config docs there. Let me also check the checkpoint.go header:


1	// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
2	// All rights reserved. This file is part of cc-backend.
3	// Use of this source code is governed by a MIT-style
4	// license that can be found in the LICENSE file.
5
6	// This file implements checkpoint persistence for the in-memory metric store.
7	//
8	// Checkpoints enable graceful restarts by periodically saving in-memory metric
9	// data to disk in JSON or binary format. The checkpoint system:
10	//
11	// Key Features:
12	//   - Periodic background checkpointing via the Checkpointing() worker
13	//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
14	//   - Parallel checkpoint creation and loading using worker pools
15	//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
16	//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
17	//   - Only saves unarchived data (archived data is already persisted elsewhere)
18	//   - GC optimization during loading to prevent excessive heap growth
19	//
20	// Checkpoint Workflow:
21	//  1. Init() loads checkpoints within retention window at startup
22	//  2. Checkpointing() worker periodically saves new data
23	//  3. Shutdown() writes final checkpoint before exit
24	//
25	// File Organization:
26	//
27	//	checkpoints/
28	//	  cluster1/
29	//	    host001/
30	//	      1234567890.json  (JSON format: full subtree snapshot)
31	//	      1234567890.bin   (binary format: full subtree snapshot)
32	//	      current.wal      (WAL format: append-only per-entry log)
33	//	    host002/
34	//	      ...
35	package metricstore

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements checkpoint persistence for the in-memory metric store.
//
// Checkpoints enable graceful restarts by periodically saving in-memory metric
// data to disk in JSON or binary format. The checkpoint system:
//
// Key Features:
//   - Periodic background checkpointing via the Checkpointing() worker
//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
//   - Parallel checkpoint creation and loading using worker pools
//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
//   - Only saves unarchived data (archived data is already persisted elsewhere)
//   - GC optimization during loading to prevent excessive heap growth
//
// Checkpoint Workflow:
//  1. Init() loads checkpoints within retention window at startup
//  2. Checkpointing() worker periodically saves new data
//  3. Shutdown() writes final checkpoint before exit
//
// File Organization:
//
//	checkpoints/
//	  cluster1/
//	    host001/
//	      1234567890.json  (JSON format: full subtree snapshot)
//	      1234567890.bin   (binary format: full subtree snapshot)
//	      current.wal      (WAL format: append-only per-entry log)
//	    host002/
//	      ...
package metricstore


Now let me verify the build still passes after the schema change:


Bash

Build after schema update


Updated:

1. **`pkg/metricstore/configSchema.go`** — Added `max-wal-size` field to the `checkpoints` JSON schema with description, type `integer`, and `minimum: 0`.

The Go struct docs in `config.go` were already updated in the previous round. The example `configs/config.json` intentionally omits it since the default (0 = unlimited) is backward compatible — users only need to add it if they want to enable the limit.


Also update the ReleaseNotes with the recent changes


1	# `cc-backend` version 1.5.3
2
3	Supports job archive version 3 and database version 11.
4
5	This is a bugfix release of `cc-backend`, the API backend and frontend
6	implementation of ClusterCockpit.
7	For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/).
8	If you are upgrading from v1.5.1 no database migration is required.
9	If you are upgrading from v1.5.0 you need to do another DB migration. This
10	should not take long. For optimal database performance after the migration it is
11	recommended to apply the new `optimize-db` flag, which runs the sqlite `ANALYZE`
12	and `VACUUM` commands. Depending on your database size (more then 40GB) the
13	`VACUUM` may take up to 2h. You can also run the `ANALYZE` command manually.
14	While we are confident that the memory issue with the metricstore cleanup move
15	policy is fixed, it is still recommended to use delete policy for cleanup.
16	This is also the default.
17
18	## Changes in 1.5.3
19
20	### Bug fixes
21
22	- **Doubleranged filter fixes**: Range filters now correctly handle zero as a
23	  boundary value. Improved validation and UI text for "more than equal" and
24	  "less than equal" range selections.
25	- **Lineprotocol body parsing interrupted**: Switched from `ReadTimeout` to
26	  `ReadHeaderTimeout` so that long-running metric submissions are no longer
27	  cut off mid-stream.
28	- **Checkpoint archiving continues on error**: A single cluster's archiving
29	  failure no longer aborts the entire cleanup operation. Errors are collected
30	  and reported per cluster.
31	- **Parquet row group overflow**: Added periodic flush during checkpoint
32	  archiving to prevent exceeding the parquet-go 32k column-write limit.
33	- **Removed metrics excluded from subcluster config**: Metrics removed from a
34	  subcluster are no longer returned by `GetMetricConfigSubCluster`.
35
36	### MetricStore performance
37
38	- **WAL writer throughput**: Decoupled WAL file flushing from message processing
39	  using a periodic 5-second batch flush (up to 4096 messages per cycle),
40	  significantly increasing metric ingestion throughput.
41	- **Improved shutdown time**: HTTP shutdown timeout reduced; metricstore and
42	  archiver now shut down concurrently. Overall shutdown deadline raised to
43	  60 seconds.
44
45	### New features
46
47	- **Manual checkpoint cleanup flag**: New `-cleanup-checkpoints` CLI flag
48	  triggers checkpoint cleanup without starting the server, useful for
49	  maintenance windows or automated cleanup scripts.
50	- **Explicit node state queries in node view**: Node health and scheduler state
51	  are now fetched independently from metric data for fresher status information.
52
53	### Logging improvements
54
55	- **Reduced tagger log noise**: Missing metrics and expression evaluation errors
56	  in the job classification tagger are now logged at debug level instead of
57	  error level.
58
59	## Changes in 1.5.2
60

# `cc-backend` version 1.5.3

Supports job archive version 3 and database version 11.

This is a bugfix release of `cc-backend`, the API backend and frontend
implementation of ClusterCockpit.
For release specific notes visit the [ClusterCockpit Documentation](https://clusterockpit.org/docs/release/).
If you are upgrading from v1.5.1 no database migration is required.
If you are upgrading from v1.5.0 you need to do another DB migration. This
should not take long. For optimal database performance after the migration it is
recommended to apply the new `optimize-db` flag, which runs the sqlite `ANALYZE`
and `VACUUM` commands. Depending on your database size (more then 40GB) the
`VACUUM` may take up to 2h. You can also run the `ANALYZE` command manually.
While we are confident that the memory issue with the metricstore cleanup move
policy is fixed, it is still recommended to use delete policy for cleanup.
This is also the default.

## Changes in 1.5.3

### Bug fixes

- **Doubleranged filter fixes**: Range filters now correctly handle zero as a
  boundary value. Improved validation and UI text for "more than equal" and
  "less than equal" range selections.
- **Lineprotocol body parsing interrupted**: Switched from `ReadTimeout` to
  `ReadHeaderTimeout` so that long-running metric submissions are no longer
  cut off mid-stream.
- **Checkpoint archiving continues on error**: A single cluster's archiving
  failure no longer aborts the entire cleanup operation. Errors are collected
  and reported per cluster.
- **Parquet row group overflow**: Added periodic flush during checkpoint
  archiving to prevent exceeding the parquet-go 32k column-write limit.
- **Removed metrics excluded from subcluster config**: Metrics removed from a
  subcluster are no longer returned by `GetMetricConfigSubCluster`.

### MetricStore performance

- **WAL writer throughput**: Decoupled WAL file flushing from message processing
  using a periodic 5-second batch flush (up to 4096 messages per cycle),
  significantly increasing metric ingestion throughput.
- **Improved shutdown time**: HTTP shutdown timeout reduced; metricstore and
  archiver now shut down concurrently. Overall shutdown deadline raised to
  60 seconds.

### New features

- **Manual checkpoint cleanup flag**: New `-cleanup-checkpoints` CLI flag
  triggers checkpoint cleanup without starting the server, useful for
  maintenance windows or automated cleanup scripts.
- **Explicit node state queries in node view**: Node health and scheduler state
  are now fetched independently from metric data for fresher status information.

### Logging improvements

- **Reduced tagger log noise**: Missing metrics and expression evaluation errors
  in the job classification tagger are now logged at debug level instead of
  error level.

## Changes in 1.5.2


Edit

## Changes in 1.5.3

### Bug fixes

- **WAL not rotated on partial checkpoint failure**: When binary checkpointing
  failed for some hosts, WAL files for successfully checkpointed hosts were not
  rotated and the checkpoint timestamp was not advanced. Partial successes now
  correctly advance the checkpoint and rotate WAL files for completed hosts.
- **Unbounded WAL file growth**: If binary checkpointing consistently failed for
  a host, its `current.wal` file grew without limit until disk exhaustion. A new
  `max-wal-size` configuration option (in the `checkpoints` block) allows setting
  a per-host WAL size cap in bytes. When exceeded, the WAL is force-rotated.
  Defaults to 0 (unlimited) for backward compatibility.