1	// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
2	// All rights reserved. This file is part of cc-backend.
3	// Use of this source code is governed by a MIT-style
4	// license that can be found in the LICENSE file.
5
6	// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
7	//
8	// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
9	// per host, eliminating the extra memory overhead of the AvroStore and providing
10	// truly continuous (per-write) crash safety.
11	//
12	// # Architecture
13	//
14	//	Metric write (DecodeLine)
15	//	      │
16	//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
17	//	      │
18	//	      └─► WALMessages channel
19	//	               │
20	//	               ▼
21	//	        WALStaging goroutine
22	//	               │
23	//	               ▼
24	//	        checkpoints/cluster/host/current.wal  (append-only, binary)
25	//
26	//	Periodic checkpoint (Checkpointing goroutine):
27	//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
28	//	  2. Signal WALStaging to truncate current.wal per host
29	//
30	//	On restart (FromCheckpoint):
31	//	  1. Load most recent <timestamp>.bin snapshot
32	//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
33	//
34	// # WAL Record Format
35	//
36	//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
37	//
38	//	payload:
39	//	  [8B timestamp int64]
40	//	  [2B metric_name_len uint16][N metric name bytes]
41	//	  [1B selector_count uint8]
42	//	  per selector: [1B selector_len uint8][M selector bytes]
43	//	  [4B value float32 bits]
44	//
45	// # Binary Snapshot Format
46	//
47	//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
48	//	Level tree (recursive):
49	//	  [4B num_metrics uint32]
50	//	  per metric:
51	//	    [2B name_len uint16][N name bytes]
52	//	    [8B frequency int64][8B start int64]
53	//	    [4B num_values uint32][num_values × 4B float32]
54	//	  [4B num_children uint32]
55	//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
56	package metricstore
57
58	import (
59		"bufio"
60		"context"
61		"encoding/binary"
62		"fmt"
63		"hash/crc32"
64		"hash/fnv"
65		"io"
66		"math"
67		"os"
68		"path"
69		"strings"
70		"sync"
71		"sync/atomic"
72		"time"
73
74		cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
75		"github.com/ClusterCockpit/cc-lib/v2/schema"
76	)
77
78	// Magic numbers for binary formats.
79	const (
80		walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
81		walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
82		snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
83	)
84
85	// walShardChs holds per-shard channels for WAL messages.
86	// Initialized by WALStaging; nil when WAL is not active.
87	var walShardChs []chan *WALMessage
88
89	// walShardRotateChs holds per-shard channels for WAL rotation requests.
90	var walShardRotateChs []chan walRotateReq
91
92	// walNumShards stores the number of shards (set during WALStaging init).
93	var walNumShards int
94
95	// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
96	var walStagingWg sync.WaitGroup
97
98	// walShuttingDown is set before closing shard channels to prevent
99	// SendWALMessage from sending on a closed channel (which panics in Go).
100	var walShuttingDown atomic.Bool
101
102	// walCheckpointActive is set during binary checkpoint writes.
103	// While active, SendWALMessage skips sending (returns true) because the
104	// snapshot captures all in-memory data, making WAL writes redundant.
105	var walCheckpointActive atomic.Bool
106
107	// WALMessage represents a single metric write to be appended to the WAL.
108	// Cluster and Node are NOT stored in the WAL record (inferred from file path).
109	type WALMessage struct {
110		MetricName string
111		Cluster    string
112		Node       string
113		Selector   []string
114		Value      schema.Float
115		Timestamp  int64
116	}
117
118	// walRotateReq requests WAL file rotation for a specific host directory.
119	// The done channel is closed by the WAL goroutine when rotation is complete.
120	type walRotateReq struct {
121		hostDir string
122		done    chan struct{}
123	}
124
125	// walFileState holds an open WAL file handle and buffered writer for one host directory.
126	type walFileState struct {
127		f     *os.File
128		w     *bufio.Writer
129		dirty bool
130		size  int64 // approximate bytes written (tracked from open + writes)
131	}
132
133	// walFlushInterval controls how often dirty WAL files are flushed to disk.
134	// Decoupling flushes from message processing lets the consumer run at memory
135	// speed, amortizing syscall overhead across many writes.
136	const walFlushInterval = 1 * time.Second
137
138	// walShardIndex computes which shard a message belongs to based on cluster+node.
139	// Uses FNV-1a hash for fast, well-distributed mapping.
140	func walShardIndex(cluster, node string) int {
141		h := fnv.New32a()
142		h.Write([]byte(cluster))
143		h.Write([]byte{0})
144		h.Write([]byte(node))
145		return int(h.Sum32()) % walNumShards
146	}
147
148	// SendWALMessage routes a WAL message to the appropriate shard channel.
149	// Returns false if the channel is full or shutdown is in progress.
150	func SendWALMessage(msg *WALMessage) bool {
151		if walShardChs == nil || walShuttingDown.Load() {
152			return false
153		}
154		if walCheckpointActive.Load() {
155			return true // Data safe in memory; snapshot will capture it
156		}
157		shard := walShardIndex(msg.Cluster, msg.Node)
158		select {
159		case walShardChs[shard] <- msg:
160			return true
161		default:
162			return false
163		}
164	}
165
166	// WALStaging starts N sharded background goroutines that receive WALMessage items
167	// and append binary WAL records to per-host current.wal files.
168	// Each shard owns a subset of hosts (determined by hash of cluster+node),
169	// parallelizing serialization and I/O while keeping per-host writes serial.
170	func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
171		if Keys.Checkpoints.FileFormat == "json" {
172			return
173		}
174
175		walNumShards = max(Keys.NumWorkers, 1)
176		chBufSize := max(65536/walNumShards, 1024)
177
178		walShardChs = make([]chan *WALMessage, walNumShards)
179		walShardRotateChs = make([]chan walRotateReq, walNumShards)
180
181		for i := range walNumShards {
182			walShardChs[i] = make(chan *WALMessage, chBufSize)
183			walShardRotateChs[i] = make(chan walRotateReq, 64)
184		}
185
186		for i := range walNumShards {
187			msgCh := walShardChs[i]
188			rotateCh := walShardRotateChs[i]
189
190			walStagingWg.Add(1)
191			wg.Go(func() {
192				defer walStagingWg.Done()
193				hostFiles := make(map[string]*walFileState)
194
195				defer func() {
196					for _, ws := range hostFiles {
197						if ws.f != nil {
198							ws.w.Flush()
199							ws.f.Close()
200						}
201					}
202				}()
203
204				getOrOpenWAL := func(hostDir string) *walFileState {
205					ws, ok := hostFiles[hostDir]
206					if ok {
207						return ws
208					}
209
210					if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
211						cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
212						return nil
213					}
214
215					walPath := path.Join(hostDir, "current.wal")
216					f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
217					if err != nil {
218						cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
219						return nil
220					}
221
222					w := bufio.NewWriter(f)
223
224					// Write file header magic if file is new (empty).
225					info, err := f.Stat()
226					var fileSize int64
227					if err == nil {
228						fileSize = info.Size()
229					}
230					if err == nil && fileSize == 0 {
231						var hdr [4]byte
232						binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
233						if _, err := w.Write(hdr[:]); err != nil {
234							cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
235							f.Close()
236							return nil
237						}
238						fileSize = 4
239					}
240
241					ws = &walFileState{f: f, w: w, size: fileSize}
242					hostFiles[hostDir] = ws
243					return ws
244				}
245
246				processMsg := func(msg *WALMessage) {
247					hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
248					ws := getOrOpenWAL(hostDir)
249					if ws == nil {
250						return
251					}
252
253					// Enforce max WAL size: force-rotate before writing if limit is exceeded.
254					// The in-memory store still holds the data; only crash-recovery coverage is lost.
255					if maxSize := Keys.Checkpoints.MaxWALSize; maxSize > 0 && ws.size >= maxSize {
256						cclog.Warnf("[METRICSTORE]> WAL: force-rotating %s (size %d >= limit %d)",
257							hostDir, ws.size, maxSize)
258						ws.w.Flush()
259						ws.f.Close()
260						walPath := path.Join(hostDir, "current.wal")
261						if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
262							cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
263						}
264						delete(hostFiles, hostDir)
265						ws = getOrOpenWAL(hostDir)
266						if ws == nil {
267							return
268						}
269					}
270
271					n, err := writeWALRecordDirect(ws.w, msg)
272					if err != nil {
273						cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
274					}
275					ws.size += int64(n)
276					ws.dirty = true
277				}
278
279				processRotate := func(req walRotateReq) {
280					ws, ok := hostFiles[req.hostDir]
281					if ok && ws.f != nil {
282						ws.w.Flush()
283						ws.f.Close()
284						walPath := path.Join(req.hostDir, "current.wal")
285						if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
286							cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
287						}
288						delete(hostFiles, req.hostDir)
289					}
290					close(req.done)
291				}
292
293				flushDirty := func() {
294					for _, ws := range hostFiles {
295						if ws.dirty {
296							ws.w.Flush()
297							ws.dirty = false
298						}
299					}
300				}
301
302				ticker := time.NewTicker(walFlushInterval)
303				defer ticker.Stop()
304
305				// drainBatch processes up to 4096 pending messages without blocking.
306				// Returns false if the channel was closed.
307				drainBatch := func() bool {
308					for range 4096 {
309						select {
310						case msg, ok := <-msgCh:
311							if !ok {
312								flushDirty()
313								return false
314							}
315							processMsg(msg)
316						case req := <-rotateCh:
317							processRotate(req)
318						default:
319							return true
320						}
321					}
322					return true
323				}
324
325				for {
326					select {
327					case <-ctx.Done():
328						// On shutdown, skip draining buffered messages — a full binary
329						// checkpoint will be written from in-memory state, making
330						// buffered WAL records redundant.
331						flushDirty()
332						return
333					case msg, ok := <-msgCh:
334						if !ok {
335							return
336						}
337						processMsg(msg)
338						if !drainBatch() {
339							return
340						}
341						// No flush here — timer handles periodic flushing.
342					case <-ticker.C:
343						flushDirty()
344					case req := <-rotateCh:
345						processRotate(req)
346					}
347				}
348			})
349		}
350	}
351
352	// WaitForWALStagingDrain blocks until all WALStaging goroutines have exited.
353	// Must be called after closing walShardChs to ensure all file handles are
354	// flushed and closed before checkpoint writes begin.
355	func WaitForWALStagingDrain() {
356		walStagingWg.Wait()
357	}
358
359	// RotateWALFiles sends rotation requests for the given host directories
360	// and blocks until all rotations complete. Each request is routed to the
361	// shard that owns the host directory.
362	//
363	// If shutdown is in progress (WAL staging goroutines may have exited),
364	// rotation is skipped to avoid deadlocking on abandoned channels.
365	func RotateWALFiles(hostDirs []string) {
366		if walShardRotateChs == nil || walShuttingDown.Load() {
367			return
368		}
369		deadline := time.After(2 * time.Minute)
370		dones := make([]chan struct{}, 0, len(hostDirs))
371		for _, dir := range hostDirs {
372			done := make(chan struct{})
373			shard := walShardIndexFromDir(dir)
374			select {
375			case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
376				dones = append(dones, done)
377			case <-deadline:
378				cclog.Warnf("[METRICSTORE]> WAL rotation send timed out, %d of %d hosts remaining",
379					len(hostDirs)-len(dones), len(hostDirs))
380				goto waitDones
381			}
382		}
383	waitDones:
384		for _, done := range dones {
385			select {
386			case <-done:
387			case <-time.After(30 * time.Second):
388				cclog.Warn("[METRICSTORE]> WAL rotation completion timed out, continuing")
389				return
390			}
391		}
392	}
393
394	// walShardIndexFromDir extracts cluster and node from a host directory path
395	// and returns the shard index. The path format is: rootDir/cluster/node
396	func walShardIndexFromDir(hostDir string) int {
397		// hostDir = .../cluster/node — extract last two path components
398		node := path.Base(hostDir)
399		cluster := path.Base(path.Dir(hostDir))
400		return walShardIndex(cluster, node)
401	}
402
403	// RotateWALFilesAfterShutdown directly removes current.wal files for the given
404	// host directories. Used after shutdown, when WALStaging goroutines have already
405	// exited and the channel-based RotateWALFiles is no longer safe to call.
406	func RotateWALFilesAfterShutdown(hostDirs []string) {
407		for _, dir := range hostDirs {
408			walPath := path.Join(dir, "current.wal")
409			if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
410				cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
411			}
412		}
413	}
414
415	// writeWALRecordDirect encodes a WAL record into a contiguous buffer first,
416	// then writes it to the bufio.Writer in a single call. This prevents partial
417	// records in the write buffer if a write error occurs mid-record (e.g. disk full).
418	// Returns the number of bytes written and any error.
419	func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) (int, error) {
420		// Compute payload and total record size.
421		payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
422		for _, s := range msg.Selector {
423			payloadSize += 1 + len(s)
424		}
425		// Total: 8 (header) + payload + 4 (CRC).
426		totalSize := 8 + payloadSize + 4
427
428		// Use stack buffer for typical small records, heap-allocate only for large ones.
429		var stackBuf [256]byte
430		var buf []byte
431		if totalSize <= len(stackBuf) {
432			buf = stackBuf[:totalSize]
433		} else {
434			buf = make([]byte, totalSize)
435		}
436
437		// Header: magic + payload length.
438		binary.LittleEndian.PutUint32(buf[0:4], walRecordMagic)
439		binary.LittleEndian.PutUint32(buf[4:8], uint32(payloadSize))
440
441		// Payload starts at offset 8.
442		p := 8
443
444		// Timestamp (8 bytes).
445		binary.LittleEndian.PutUint64(buf[p:p+8], uint64(msg.Timestamp))
446		p += 8
447
448		// Metric name length (2 bytes) + metric name.
449		binary.LittleEndian.PutUint16(buf[p:p+2], uint16(len(msg.MetricName)))
450		p += 2
451		p += copy(buf[p:], msg.MetricName)
452
453		// Selector count (1 byte).
454		buf[p] = byte(len(msg.Selector))
455		p++
456
457		// Selectors (1-byte length + bytes each).
458		for _, sel := range msg.Selector {
459			buf[p] = byte(len(sel))
460			p++
461			p += copy(buf[p:], sel)
462		}
463
464		// Value (4 bytes, float32 bits).
465		binary.LittleEndian.PutUint32(buf[p:p+4], math.Float32bits(float32(msg.Value)))
466		p += 4
467
468		// CRC32 over payload (bytes 8..8+payloadSize).
469		crc := crc32.ChecksumIEEE(buf[8 : 8+payloadSize])
470		binary.LittleEndian.PutUint32(buf[p:p+4], crc)
471
472		// Single atomic write to the buffered writer.
473		n, err := w.Write(buf)
474		return n, err
475	}
476
477	// readWALRecord reads one WAL record from the reader.
478	// Returns (nil, nil) on clean EOF. Returns error on data corruption.
479	// A CRC mismatch indicates a truncated trailing record (expected on crash).
480	func readWALRecord(r io.Reader) (*WALMessage, error) {
481		var magic uint32
482		if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
483			if err == io.EOF {
484				return nil, nil // Clean EOF
485			}
486			return nil, fmt.Errorf("read record magic: %w", err)
487		}
488
489		if magic != walRecordMagic {
490			return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
491		}
492
493		var payloadLen uint32
494		if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
495			return nil, fmt.Errorf("read payload length: %w", err)
496		}
497
498		if payloadLen > 1<<20 { // 1 MB sanity limit
499			return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
500		}
501
502		payload := make([]byte, payloadLen)
503		if _, err := io.ReadFull(r, payload); err != nil {
504			return nil, fmt.Errorf("read payload: %w", err)
505		}
506
507		var storedCRC uint32
508		if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
509			return nil, fmt.Errorf("read CRC: %w", err)
510		}
511
512		if crc32.ChecksumIEEE(payload) != storedCRC {
513			return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
514		}
515
516		return parseWALPayload(payload)
517	}
518
519	// parseWALPayload decodes a binary payload into a WALMessage.
520	func parseWALPayload(payload []byte) (*WALMessage, error) {
521		if len(payload) < 8+2+1+4 {
522			return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
523		}
524
525		offset := 0
526
527		// Timestamp (8 bytes)
528		ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
529		offset += 8
530
531		// Metric name (2-byte length + bytes)
532		if offset+2 > len(payload) {
533			return nil, fmt.Errorf("metric name length overflows payload")
534		}
535		mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
536		offset += 2
537
538		if offset+mLen > len(payload) {
539			return nil, fmt.Errorf("metric name overflows payload")
540		}
541		metricName := string(payload[offset : offset+mLen])
542		offset += mLen
543
544		// Selector count (1 byte)
545		if offset >= len(payload) {
546			return nil, fmt.Errorf("selector count overflows payload")
547		}
548		selCount := int(payload[offset])
549		offset++
550
551		selectors := make([]string, selCount)
552		for i := range selCount {
553			if offset >= len(payload) {
554				return nil, fmt.Errorf("selector[%d] length overflows payload", i)
555			}
556			sLen := int(payload[offset])
557			offset++
558
559			if offset+sLen > len(payload) {
560				return nil, fmt.Errorf("selector[%d] data overflows payload", i)
561			}
562			selectors[i] = string(payload[offset : offset+sLen])
563			offset += sLen
564		}
565
566		// Value (4 bytes, float32 bits)
567		if offset+4 > len(payload) {
568			return nil, fmt.Errorf("value overflows payload")
569		}
570		bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
571		value := schema.Float(math.Float32frombits(bits))
572
573		return &WALMessage{
574			MetricName: metricName,
575			Timestamp:  ts,
576			Selector:   selectors,
577			Value:      value,
578		}, nil
579	}
580
581	// loadWALFile reads a WAL file and replays all valid records into the Level tree.
582	// l is the host-level node. Corrupt or partial trailing records are silently skipped
583	// (expected on crash). Records older than 'from' are skipped.
584	func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
585		br := bufio.NewReader(f)
586
587		// Verify file header magic.
588		var fileMagic uint32
589		if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
590			if err == io.EOF {
591				return nil // Empty file, no data
592			}
593			return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
594		}
595
596		if fileMagic != walFileMagic {
597			return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
598		}
599
600		// Cache level lookups to avoid repeated tree traversal.
601		lvlCache := make(map[string]*Level)
602
603		for {
604			msg, err := readWALRecord(br)
605			if err != nil {
606				// Truncated trailing record is expected after a crash; stop replaying.
607				cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
608				break
609			}
610			if msg == nil {
611				break // Clean EOF
612			}
613
614			if msg.Timestamp < from {
615				continue // Older than retention window
616			}
617
618			minfo, ok := m.Metrics[msg.MetricName]
619			if !ok {
620				continue // Unknown metric (config may have changed)
621			}
622
623			// Cache key is the null-separated selector path.
624			cacheKey := joinSelector(msg.Selector)
625			lvl, ok := lvlCache[cacheKey]
626			if !ok {
627				lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
628				lvlCache[cacheKey] = lvl
629			}
630
631			// Write directly to the buffer, same as WriteToLevel but without the
632			// global level lookup (we already have the right level).
633			lvl.lock.Lock()
634			b := lvl.metrics[minfo.offset]
635			if b == nil {
636				b = newBuffer(msg.Timestamp, minfo.Frequency)
637				lvl.metrics[minfo.offset] = b
638			}
639			nb, writeErr := b.write(msg.Timestamp, msg.Value)
640			if writeErr == nil && b != nb {
641				lvl.metrics[minfo.offset] = nb
642			}
643			// Ignore write errors for timestamps before buffer start (can happen when
644			// replaying WAL entries that predate a loaded snapshot's start time).
645			lvl.lock.Unlock()
646		}
647
648		return nil
649	}
650
651	// joinSelector builds a cache key from a selector slice using null bytes as separators.
652	func joinSelector(sel []string) string {
653		if len(sel) == 0 {
654			return ""
655		}
656		var result strings.Builder
657		result.WriteString(sel[0])
658		for i := 1; i < len(sel); i++ {
659			result.WriteString("\x00" + sel[i])
660		}
661		return result.String()
662	}
663
664	// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
665	// Returns the number of files written, the list of host directories that were
666	// successfully checkpointed (for WAL rotation), and any errors.
667	func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
668		// Collect all cluster/host pairs.
669		m.root.lock.RLock()
670		totalHosts := 0
671		for _, l1 := range m.root.children {
672			l1.lock.RLock()
673			totalHosts += len(l1.children)
674			l1.lock.RUnlock()
675		}
676		m.root.lock.RUnlock()
677
678		levels := make([]*Level, 0, totalHosts)
679		selectors := make([][]string, 0, totalHosts)
680
681		m.root.lock.RLock()
682		for sel1, l1 := range m.root.children {
683			l1.lock.RLock()
684			for sel2, l2 := range l1.children {
685				levels = append(levels, l2)
686				selectors = append(selectors, []string{sel1, sel2})
687			}
688			l1.lock.RUnlock()
689		}
690		m.root.lock.RUnlock()
691
692		type workItem struct {
693			level    *Level
694			hostDir  string
695			selector []string
696		}
697
698		totalWork := len(levels)
699		cclog.Infof("[METRICSTORE]> Starting binary checkpoint for %d hosts with %d workers", totalWork, Keys.NumWorkers)
700
701		n, errs, completed := int32(0), int32(0), int32(0)
702		var successDirs []string
703		var successMu sync.Mutex
704
705		var wg sync.WaitGroup
706		wg.Add(Keys.NumWorkers)
707		work := make(chan workItem, Keys.NumWorkers*2)
708
709		// Progress logging goroutine.
710		stopProgress := make(chan struct{})
711		go func() {
712			ticker := time.NewTicker(10 * time.Second)
713			defer ticker.Stop()
714			for {
715				select {
716				case <-ticker.C:
717					cclog.Infof("[METRICSTORE]> Checkpoint progress: %d/%d hosts (%d written, %d errors)",
718						atomic.LoadInt32(&completed), totalWork, atomic.LoadInt32(&n), atomic.LoadInt32(&errs))
719				case <-stopProgress:
720					return
721				}
722			}
723		}()
724
725		for range Keys.NumWorkers {
726			go func() {
727				defer wg.Done()
728				for wi := range work {
729					err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
730					if err != nil {
731						if err == ErrNoNewArchiveData {
732							atomic.AddInt32(&completed, 1)
733							continue
734						}
735						cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
736						atomic.AddInt32(&errs, 1)
737					} else {
738						atomic.AddInt32(&n, 1)
739						successMu.Lock()
740						successDirs = append(successDirs, wi.hostDir)
741						successMu.Unlock()
742					}
743					atomic.AddInt32(&completed, 1)
744				}
745			}()
746		}
747
748		for i := range levels {
749			hostDir := path.Join(dir, path.Join(selectors[i]...))
750			work <- workItem{
751				level:    levels[i],
752				hostDir:  hostDir,
753				selector: selectors[i],
754			}
755		}
756		close(work)
757		wg.Wait()
758		close(stopProgress)
759
760		if errs > 0 {
761			return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
762		}
763		return int(n), successDirs, nil
764	}
765
766	// toCheckpointBinary writes a binary snapshot file for a single host-level node.
767	// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
768	func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
769		cf, err := l.toCheckpointFile(from, to, m)
770		if err != nil {
771			return err
772		}
773		if cf == nil {
774			return ErrNoNewArchiveData
775		}
776
777		if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
778			return fmt.Errorf("mkdir %s: %w", dir, err)
779		}
780
781		// Write to a temp file first, then rename (atomic on POSIX).
782		tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
783		finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))
784
785		f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
786		if err != nil {
787			return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
788		}
789
790		bw := bufio.NewWriter(f)
791		if err := writeBinarySnapshotFile(bw, cf); err != nil {
792			f.Close()
793			os.Remove(tmpPath)
794			return fmt.Errorf("write binary snapshot: %w", err)
795		}
796		if err := bw.Flush(); err != nil {
797			f.Close()
798			os.Remove(tmpPath)
799			return err
800		}
801		f.Close()
802
803		return os.Rename(tmpPath, finalPath)
804	}
805
806	// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
807	func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
808		if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
809			return err
810		}
811		if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
812			return err
813		}
814		if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
815			return err
816		}
817		return writeBinaryLevel(w, cf)
818	}
819
820	// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
821	func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
822		if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
823			return err
824		}
825
826		for name, metric := range cf.Metrics {
827			if err := writeString16(w, name); err != nil {
828				return err
829			}
830			if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
831				return err
832			}
833			if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
834				return err
835			}
836			if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
837				return err
838			}
839			for _, v := range metric.Data {
840				if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
841					return err
842				}
843			}
844		}
845
846		if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
847			return err
848		}
849
850		for name, child := range cf.Children {
851			if err := writeString16(w, name); err != nil {
852				return err
853			}
854			if err := writeBinaryLevel(w, child); err != nil {
855				return err
856			}
857		}
858
859		return nil
860	}
861
862	// writeString16 writes a 2-byte length-prefixed string to w.
863	func writeString16(w io.Writer, s string) error {
864		if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
865			return err
866		}
867		_, err := io.WriteString(w, s)
868		return err
869	}
870
871	// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
872	// The retention check (from) is applied to the file's 'to' timestamp.
873	func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
874		br := bufio.NewReader(f)
875
876		var magic uint32
877		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
878			return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
879		}
880		if magic != snapFileMagic {
881			return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
882		}
883
884		var fileFrom, fileTo int64
885		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
886			return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
887		}
888		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
889			return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
890		}
891
892		if fileTo != 0 && fileTo < from {
893			return nil // File is older than retention window, skip it
894		}
895
896		cf, err := readBinaryLevel(br)
897		if err != nil {
898			return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
899		}
900		cf.From = fileFrom
901		cf.To = fileTo
902
903		return l.loadFile(cf, m)
904	}
905
906	// readBinaryLevel recursively reads a level from the binary snapshot format.
907	func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
908		cf := &CheckpointFile{
909			Metrics:  make(map[string]*CheckpointMetrics),
910			Children: make(map[string]*CheckpointFile),
911		}
912
913		var numMetrics uint32
914		if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
915			return nil, fmt.Errorf("read num_metrics: %w", err)
916		}
917
918		for range numMetrics {
919			name, err := readString16(r)
920			if err != nil {
921				return nil, fmt.Errorf("read metric name: %w", err)
922			}
923
924			var freq, start int64
925			if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
926				return nil, fmt.Errorf("read frequency for %s: %w", name, err)
927			}
928			if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
929				return nil, fmt.Errorf("read start for %s: %w", name, err)
930			}
931
932			var numValues uint32
933			if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
934				return nil, fmt.Errorf("read num_values for %s: %w", name, err)
935			}
936
937			data := make([]schema.Float, numValues)
938			for i := range numValues {
939				var bits uint32
940				if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
941					return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
942				}
943				data[i] = schema.Float(math.Float32frombits(bits))
944			}
945
946			cf.Metrics[name] = &CheckpointMetrics{
947				Frequency: freq,
948				Start:     start,
949				Data:      data,
950			}
951		}
952
953		var numChildren uint32
954		if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
955			return nil, fmt.Errorf("read num_children: %w", err)
956		}
957
958		for range numChildren {
959			childName, err := readString16(r)
960			if err != nil {
961				return nil, fmt.Errorf("read child name: %w", err)
962			}
963
964			child, err := readBinaryLevel(r)
965			if err != nil {
966				return nil, fmt.Errorf("read child %s: %w", childName, err)
967			}
968			cf.Children[childName] = child
969		}
970
971		return cf, nil
972	}
973
974	// readString16 reads a 2-byte length-prefixed string from r.
975	func readString16(r io.Reader) (string, error) {
976		var sLen uint16
977		if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
978			return "", err
979		}
980		buf := make([]byte, sLen)
981		if _, err := io.ReadFull(r, buf); err != nil {
982			return "", err
983		}
984		return string(buf), nil
985	}
986

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼
//	        WALStaging goroutine
//	               │
//	               ▼
//	        checkpoints/cluster/host/current.wal  (append-only, binary)
//
//	Periodic checkpoint (Checkpointing goroutine):
//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
//	  2. Signal WALStaging to truncate current.wal per host
//
//	On restart (FromCheckpoint):
//	  1. Load most recent <timestamp>.bin snapshot
//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
//
// # WAL Record Format
//
//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
//
//	payload:
//	  [8B timestamp int64]
//	  [2B metric_name_len uint16][N metric name bytes]
//	  [1B selector_count uint8]
//	  per selector: [1B selector_len uint8][M selector bytes]
//	  [4B value float32 bits]
//
// # Binary Snapshot Format
//
//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
//	Level tree (recursive):
//	  [4B num_metrics uint32]
//	  per metric:
//	    [2B name_len uint16][N name bytes]
//	    [8B frequency int64][8B start int64]
//	    [4B num_values uint32][num_values × 4B float32]
//	  [4B num_children uint32]
//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
package metricstore

import (
	"bufio"
	"context"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"hash/fnv"
	"io"
	"math"
	"os"
	"path"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Magic numbers for binary formats.
const (
	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
)

// walShardChs holds per-shard channels for WAL messages.
// Initialized by WALStaging; nil when WAL is not active.
var walShardChs []chan *WALMessage

// walShardRotateChs holds per-shard channels for WAL rotation requests.
var walShardRotateChs []chan walRotateReq

// walNumShards stores the number of shards (set during WALStaging init).
var walNumShards int

// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
var walStagingWg sync.WaitGroup

// walShuttingDown is set before closing shard channels to prevent
// SendWALMessage from sending on a closed channel (which panics in Go).
var walShuttingDown atomic.Bool

// walCheckpointActive is set during binary checkpoint writes.
// While active, SendWALMessage skips sending (returns true) because the
// snapshot captures all in-memory data, making WAL writes redundant.
var walCheckpointActive atomic.Bool

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f     *os.File
	w     *bufio.Writer
	dirty bool
	size  int64 // approximate bytes written (tracked from open + writes)
}

// walFlushInterval controls how often dirty WAL files are flushed to disk.
// Decoupling flushes from message processing lets the consumer run at memory
// speed, amortizing syscall overhead across many writes.
const walFlushInterval = 1 * time.Second

// walShardIndex computes which shard a message belongs to based on cluster+node.
// Uses FNV-1a hash for fast, well-distributed mapping.
func walShardIndex(cluster, node string) int {
	h := fnv.New32a()
	h.Write([]byte(cluster))
	h.Write([]byte{0})
	h.Write([]byte(node))
	return int(h.Sum32()) % walNumShards
}

// SendWALMessage routes a WAL message to the appropriate shard channel.
// Returns false if the channel is full or shutdown is in progress.
func SendWALMessage(msg *WALMessage) bool {
	if walShardChs == nil || walShuttingDown.Load() {
		return false
	}
	if walCheckpointActive.Load() {
		return true // Data safe in memory; snapshot will capture it
	}
	shard := walShardIndex(msg.Cluster, msg.Node)
	select {
	case walShardChs[shard] <- msg:
		return true
	default:
		return false
	}
}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.
// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return
	}

	walNumShards = max(Keys.NumWorkers, 1)
	chBufSize := max(65536/walNumShards, 1024)

	walShardChs = make([]chan *WALMessage, walNumShards)
	walShardRotateChs = make([]chan walRotateReq, walNumShards)

	for i := range walNumShards {
		walShardChs[i] = make(chan *WALMessage, chBufSize)
		walShardRotateChs[i] = make(chan walRotateReq, 64)
	}

	for i := range walNumShards {
		msgCh := walShardChs[i]
		rotateCh := walShardRotateChs[i]

		walStagingWg.Add(1)
		wg.Go(func() {
			defer walStagingWg.Done()
			hostFiles := make(map[string]*walFileState)

			defer func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
						ws.f.Close()
					}
				}
			}()

			getOrOpenWAL := func(hostDir string) *walFileState {
				ws, ok := hostFiles[hostDir]
				if ok {
					return ws
				}

				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
					return nil
				}

				walPath := path.Join(hostDir, "current.wal")
				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
					return nil
				}

				w := bufio.NewWriter(f)

				// Write file header magic if file is new (empty).
				info, err := f.Stat()
				var fileSize int64
				if err == nil {
					fileSize = info.Size()
				}
				if err == nil && fileSize == 0 {
					var hdr [4]byte
					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
					if _, err := w.Write(hdr[:]); err != nil {
						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
						f.Close()
						return nil
					}
					fileSize = 4
				}

				ws = &walFileState{f: f, w: w, size: fileSize}
				hostFiles[hostDir] = ws
				return ws
			}

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}

				// Enforce max WAL size: force-rotate before writing if limit is exceeded.
				// The in-memory store still holds the data; only crash-recovery coverage is lost.
				if maxSize := Keys.Checkpoints.MaxWALSize; maxSize > 0 && ws.size >= maxSize {
					cclog.Warnf("[METRICSTORE]> WAL: force-rotating %s (size %d >= limit %d)",
						hostDir, ws.size, maxSize)
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, hostDir)
					ws = getOrOpenWAL(hostDir)
					if ws == nil {
						return
					}
				}

				n, err := writeWALRecordDirect(ws.w, msg)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
				ws.size += int64(n)
				ws.dirty = true
			}

			processRotate := func(req walRotateReq) {
				ws, ok := hostFiles[req.hostDir]
				if ok && ws.f != nil {
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(req.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, req.hostDir)
				}
				close(req.done)
			}

			flushDirty := func() {
				for _, ws := range hostFiles {
					if ws.dirty {
						ws.w.Flush()
						ws.dirty = false
					}
				}
			}

			ticker := time.NewTicker(walFlushInterval)
			defer ticker.Stop()

			// drainBatch processes up to 4096 pending messages without blocking.
			// Returns false if the channel was closed.
			drainBatch := func() bool {
				for range 4096 {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							flushDirty()
							return false
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						return true
					}
				}
				return true
			}

			for {
				select {
				case <-ctx.Done():
					// On shutdown, skip draining buffered messages — a full binary
					// checkpoint will be written from in-memory state, making
					// buffered WAL records redundant.
					flushDirty()
					return
				case msg, ok := <-msgCh:
					if !ok {
						return
					}
					processMsg(msg)
					if !drainBatch() {
						return
					}
					// No flush here — timer handles periodic flushing.
				case <-ticker.C:
					flushDirty()
				case req := <-rotateCh:
					processRotate(req)
				}
			}
		})
	}
}

// WaitForWALStagingDrain blocks until all WALStaging goroutines have exited.
// Must be called after closing walShardChs to ensure all file handles are
// flushed and closed before checkpoint writes begin.
func WaitForWALStagingDrain() {
	walStagingWg.Wait()
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete. Each request is routed to the
// shard that owns the host directory.
//
// If shutdown is in progress (WAL staging goroutines may have exited),
// rotation is skipped to avoid deadlocking on abandoned channels.
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	deadline := time.After(2 * time.Minute)
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		case <-deadline:
			cclog.Warnf("[METRICSTORE]> WAL rotation send timed out, %d of %d hosts remaining",
				len(hostDirs)-len(dones), len(hostDirs))
			goto waitDones
		}
	}
waitDones:
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation completion timed out, continuing")
			return
		}
	}
}

// walShardIndexFromDir extracts cluster and node from a host directory path
// and returns the shard index. The path format is: rootDir/cluster/node
func walShardIndexFromDir(hostDir string) int {
	// hostDir = .../cluster/node — extract last two path components
	node := path.Base(hostDir)
	cluster := path.Base(path.Dir(hostDir))
	return walShardIndex(cluster, node)
}

// RotateWALFilesAfterShutdown directly removes current.wal files for the given
// host directories. Used after shutdown, when WALStaging goroutines have already
// exited and the channel-based RotateWALFiles is no longer safe to call.
func RotateWALFilesAfterShutdown(hostDirs []string) {
	for _, dir := range hostDirs {
		walPath := path.Join(dir, "current.wal")
		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
		}
	}
}

// writeWALRecordDirect encodes a WAL record into a contiguous buffer first,
// then writes it to the bufio.Writer in a single call. This prevents partial
// records in the write buffer if a write error occurs mid-record (e.g. disk full).
// Returns the number of bytes written and any error.
func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) (int, error) {
	// Compute payload and total record size.
	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
	for _, s := range msg.Selector {
		payloadSize += 1 + len(s)
	}
	// Total: 8 (header) + payload + 4 (CRC).
	totalSize := 8 + payloadSize + 4

	// Use stack buffer for typical small records, heap-allocate only for large ones.
	var stackBuf [256]byte
	var buf []byte
	if totalSize <= len(stackBuf) {
		buf = stackBuf[:totalSize]
	} else {
		buf = make([]byte, totalSize)
	}

	// Header: magic + payload length.
	binary.LittleEndian.PutUint32(buf[0:4], walRecordMagic)
	binary.LittleEndian.PutUint32(buf[4:8], uint32(payloadSize))

	// Payload starts at offset 8.
	p := 8

	// Timestamp (8 bytes).
	binary.LittleEndian.PutUint64(buf[p:p+8], uint64(msg.Timestamp))
	p += 8

	// Metric name length (2 bytes) + metric name.
	binary.LittleEndian.PutUint16(buf[p:p+2], uint16(len(msg.MetricName)))
	p += 2
	p += copy(buf[p:], msg.MetricName)

	// Selector count (1 byte).
	buf[p] = byte(len(msg.Selector))
	p++

	// Selectors (1-byte length + bytes each).
	for _, sel := range msg.Selector {
		buf[p] = byte(len(sel))
		p++
		p += copy(buf[p:], sel)
	}

	// Value (4 bytes, float32 bits).
	binary.LittleEndian.PutUint32(buf[p:p+4], math.Float32bits(float32(msg.Value)))
	p += 4

	// CRC32 over payload (bytes 8..8+payloadSize).
	crc := crc32.ChecksumIEEE(buf[8 : 8+payloadSize])
	binary.LittleEndian.PutUint32(buf[p:p+4], crc)

	// Single atomic write to the buffered writer.
	n, err := w.Write(buf)
	return n, err
}

// readWALRecord reads one WAL record from the reader.
// Returns (nil, nil) on clean EOF. Returns error on data corruption.
// A CRC mismatch indicates a truncated trailing record (expected on crash).
func readWALRecord(r io.Reader) (*WALMessage, error) {
	var magic uint32
	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
		if err == io.EOF {
			return nil, nil // Clean EOF
		}
		return nil, fmt.Errorf("read record magic: %w", err)
	}

	if magic != walRecordMagic {
		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
	}

	var payloadLen uint32
	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
		return nil, fmt.Errorf("read payload length: %w", err)
	}

	if payloadLen > 1<<20 { // 1 MB sanity limit
		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
	}

	payload := make([]byte, payloadLen)
	if _, err := io.ReadFull(r, payload); err != nil {
		return nil, fmt.Errorf("read payload: %w", err)
	}

	var storedCRC uint32
	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
		return nil, fmt.Errorf("read CRC: %w", err)
	}

	if crc32.ChecksumIEEE(payload) != storedCRC {
		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
	}

	return parseWALPayload(payload)
}

// parseWALPayload decodes a binary payload into a WALMessage.
func parseWALPayload(payload []byte) (*WALMessage, error) {
	if len(payload) < 8+2+1+4 {
		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
	}

	offset := 0

	// Timestamp (8 bytes)
	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
	offset += 8

	// Metric name (2-byte length + bytes)
	if offset+2 > len(payload) {
		return nil, fmt.Errorf("metric name length overflows payload")
	}
	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
	offset += 2

	if offset+mLen > len(payload) {
		return nil, fmt.Errorf("metric name overflows payload")
	}
	metricName := string(payload[offset : offset+mLen])
	offset += mLen

	// Selector count (1 byte)
	if offset >= len(payload) {
		return nil, fmt.Errorf("selector count overflows payload")
	}
	selCount := int(payload[offset])
	offset++

	selectors := make([]string, selCount)
	for i := range selCount {
		if offset >= len(payload) {
			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
		}
		sLen := int(payload[offset])
		offset++

		if offset+sLen > len(payload) {
			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
		}
		selectors[i] = string(payload[offset : offset+sLen])
		offset += sLen
	}

	// Value (4 bytes, float32 bits)
	if offset+4 > len(payload) {
		return nil, fmt.Errorf("value overflows payload")
	}
	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
	value := schema.Float(math.Float32frombits(bits))

	return &WALMessage{
		MetricName: metricName,
		Timestamp:  ts,
		Selector:   selectors,
		Value:      value,
	}, nil
}

// loadWALFile reads a WAL file and replays all valid records into the Level tree.
// l is the host-level node. Corrupt or partial trailing records are silently skipped
// (expected on crash). Records older than 'from' are skipped.
func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	// Verify file header magic.
	var fileMagic uint32
	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
		if err == io.EOF {
			return nil // Empty file, no data
		}
		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
	}

	if fileMagic != walFileMagic {
		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
	}

	// Cache level lookups to avoid repeated tree traversal.
	lvlCache := make(map[string]*Level)

	for {
		msg, err := readWALRecord(br)
		if err != nil {
			// Truncated trailing record is expected after a crash; stop replaying.
			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
			break
		}
		if msg == nil {
			break // Clean EOF
		}

		if msg.Timestamp < from {
			continue // Older than retention window
		}

		minfo, ok := m.Metrics[msg.MetricName]
		if !ok {
			continue // Unknown metric (config may have changed)
		}

		// Cache key is the null-separated selector path.
		cacheKey := joinSelector(msg.Selector)
		lvl, ok := lvlCache[cacheKey]
		if !ok {
			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
			lvlCache[cacheKey] = lvl
		}

		// Write directly to the buffer, same as WriteToLevel but without the
		// global level lookup (we already have the right level).
		lvl.lock.Lock()
		b := lvl.metrics[minfo.offset]
		if b == nil {
			b = newBuffer(msg.Timestamp, minfo.Frequency)
			lvl.metrics[minfo.offset] = b
		}
		nb, writeErr := b.write(msg.Timestamp, msg.Value)
		if writeErr == nil && b != nb {
			lvl.metrics[minfo.offset] = nb
		}
		// Ignore write errors for timestamps before buffer start (can happen when
		// replaying WAL entries that predate a loaded snapshot's start time).
		lvl.lock.Unlock()
	}

	return nil
}

// joinSelector builds a cache key from a selector slice using null bytes as separators.
func joinSelector(sel []string) string {
	if len(sel) == 0 {
		return ""
	}
	var result strings.Builder
	result.WriteString(sel[0])
	for i := 1; i < len(sel); i++ {
		result.WriteString("\x00" + sel[i])
	}
	return result.String()
}

// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
// Returns the number of files written, the list of host directories that were
// successfully checkpointed (for WAL rotation), and any errors.
func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
	// Collect all cluster/host pairs.
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		hostDir  string
		selector []string
	}

	totalWork := len(levels)
	cclog.Infof("[METRICSTORE]> Starting binary checkpoint for %d hosts with %d workers", totalWork, Keys.NumWorkers)

	n, errs, completed := int32(0), int32(0), int32(0)
	var successDirs []string
	var successMu sync.Mutex

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)

	// Progress logging goroutine.
	stopProgress := make(chan struct{})
	go func() {
		ticker := time.NewTicker(10 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				cclog.Infof("[METRICSTORE]> Checkpoint progress: %d/%d hosts (%d written, %d errors)",
					atomic.LoadInt32(&completed), totalWork, atomic.LoadInt32(&n), atomic.LoadInt32(&errs))
			case <-stopProgress:
				return
			}
		}
	}()

	for range Keys.NumWorkers {
		go func() {
			defer wg.Done()
			for wi := range work {
				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
				if err != nil {
					if err == ErrNoNewArchiveData {
						atomic.AddInt32(&completed, 1)
						continue
					}
					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
					successMu.Lock()
					successDirs = append(successDirs, wi.hostDir)
					successMu.Unlock()
				}
				atomic.AddInt32(&completed, 1)
			}
		}()
	}

	for i := range levels {
		hostDir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			hostDir:  hostDir,
			selector: selectors[i],
		}
	}
	close(work)
	wg.Wait()
	close(stopProgress)

	if errs > 0 {
		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
	}
	return int(n), successDirs, nil
}

// toCheckpointBinary writes a binary snapshot file for a single host-level node.
// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}
	if cf == nil {
		return ErrNoNewArchiveData
	}

	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
		return fmt.Errorf("mkdir %s: %w", dir, err)
	}

	// Write to a temp file first, then rename (atomic on POSIX).
	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))

	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
	}

	bw := bufio.NewWriter(f)
	if err := writeBinarySnapshotFile(bw, cf); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return fmt.Errorf("write binary snapshot: %w", err)
	}
	if err := bw.Flush(); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return err
	}
	f.Close()

	return os.Rename(tmpPath, finalPath)
}

// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
		return err
	}
	return writeBinaryLevel(w, cf)
}

// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
		return err
	}

	for name, metric := range cf.Metrics {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
			return err
		}
		for _, v := range metric.Data {
			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
				return err
			}
		}
	}

	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
		return err
	}

	for name, child := range cf.Children {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := writeBinaryLevel(w, child); err != nil {
			return err
		}
	}

	return nil
}

// writeString16 writes a 2-byte length-prefixed string to w.
func writeString16(w io.Writer, s string) error {
	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
		return err
	}
	_, err := io.WriteString(w, s)
	return err
}

// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
// The retention check (from) is applied to the file's 'to' timestamp.
func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	var magic uint32
	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
	}
	if magic != snapFileMagic {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
	}

	var fileFrom, fileTo int64
	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
	}
	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
	}

	if fileTo != 0 && fileTo < from {
		return nil // File is older than retention window, skip it
	}

	cf, err := readBinaryLevel(br)
	if err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
	}
	cf.From = fileFrom
	cf.To = fileTo

	return l.loadFile(cf, m)
}

// readBinaryLevel recursively reads a level from the binary snapshot format.
func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
	cf := &CheckpointFile{
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	var numMetrics uint32
	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
		return nil, fmt.Errorf("read num_metrics: %w", err)
	}

	for range numMetrics {
		name, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read metric name: %w", err)
		}

		var freq, start int64
		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
		}
		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
			return nil, fmt.Errorf("read start for %s: %w", name, err)
		}

		var numValues uint32
		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
		}

		data := make([]schema.Float, numValues)
		for i := range numValues {
			var bits uint32
			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
			}
			data[i] = schema.Float(math.Float32frombits(bits))
		}

		cf.Metrics[name] = &CheckpointMetrics{
			Frequency: freq,
			Start:     start,
			Data:      data,
		}
	}

	var numChildren uint32
	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
		return nil, fmt.Errorf("read num_children: %w", err)
	}

	for range numChildren {
		childName, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read child name: %w", err)
		}

		child, err := readBinaryLevel(r)
		if err != nil {
			return nil, fmt.Errorf("read child %s: %w", childName, err)
		}
		cf.Children[childName] = child
	}

	return cf, nil
}

// readString16 reads a 2-byte length-prefixed string from r.
func readString16(r io.Reader) (string, error) {
	var sLen uint16
	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
		return "", err
	}
	buf := make([]byte, sLen)
	if _, err := io.ReadFull(r, buf); err != nil {
		return "", err
	}
	return string(buf), nil
}


This application uses a files based checkpointing mechanism using a WAL algorithm for crash recovery. Analyse when the wal files are reset. Currently the wal files are not reset and continously grow a…


Analyze WAL file reset mechanism in checkpoint system


I have a thorough understanding of the WAL and checkpoint logic. Let me examine the initialization, shutdown, and config to complete the picture.