1	// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
2	// All rights reserved. This file is part of cc-backend.
3	// Use of this source code is governed by a MIT-style
4	// license that can be found in the LICENSE file.
5
6	// Package metricstore provides an efficient in-memory time-series metric storage system
7	// with support for hierarchical data organization, checkpointing, and archiving.
8	//
9	// The package organizes metrics in a tree structure (cluster → host → component) and
10	// provides concurrent read/write access to metric data with configurable aggregation strategies.
11	// Background goroutines handle periodic checkpointing (JSON or WAL/binary format), archiving old data,
12	// and enforcing retention policies.
13	//
14	// Key features:
15	//   - In-memory metric storage with configurable retention
16	//   - Hierarchical data organization (selectors)
17	//   - Concurrent checkpoint/archive workers
18	//   - Support for sum and average aggregation
19	//   - NATS integration for metric ingestion via InfluxDB line protocol
20	package metricstore
21
22	import (
23		"bytes"
24		"context"
25		"encoding/json"
26		"errors"
27		"fmt"
28		"runtime"
29		"runtime/debug"
30		"slices"
31		"sync"
32		"time"
33
34		"github.com/ClusterCockpit/cc-backend/internal/config"
35		cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
36		"github.com/ClusterCockpit/cc-lib/v2/resampler"
37		"github.com/ClusterCockpit/cc-lib/v2/schema"
38		"github.com/ClusterCockpit/cc-lib/v2/util"
39	)
40
41	// GlobalState holds the global state for the metric store with thread-safe access.
42	type GlobalState struct {
43		mu                sync.RWMutex
44		lastRetentionTime int64
45		selectorsExcluded bool
46	}
47
48	var (
49		singleton  sync.Once
50		msInstance *MemoryStore
51		// shutdownFunc stores the context cancellation function created in Init
52		// and is called during Shutdown to cancel all background goroutines
53		shutdownFunc   context.CancelFunc
54		shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
55		// Create a global instance
56		state = &GlobalState{}
57	)
58
59	// NodeProvider provides information about nodes currently in use by running jobs.
60	//
61	// This interface allows metricstore to query job information without directly
62	// depending on the repository package, breaking the import cycle.
63	//
64	// Implementations should return nodes that are actively processing jobs started
65	// before the given timestamp. These nodes will be excluded from retention-based
66	// garbage collection to prevent data loss for jobs that are still running or
67	// recently completed.
68	type NodeProvider interface {
69		// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
70		// that are currently in use by jobs that started before the given timestamp.
71		//
72		// Parameters:
73		//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
74		//
75		// Returns:
76		//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
77		//   - Error if the query fails
78		GetUsedNodes(ts int64) (map[string][]string, error)
79	}
80
81	// Metric represents a single metric data point to be written to the store.
82	type Metric struct {
83		Name  string
84		Value schema.Float
85		// MetricConfig contains frequency and aggregation settings for this metric.
86		// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
87		MetricConfig MetricConfig
88	}
89
90	// MemoryStore is the main in-memory time-series metric storage implementation.
91	//
92	// It organizes metrics in a hierarchical tree structure where each level represents
93	// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
94	// store multiple metrics as time-series buffers.
95	//
96	// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
97	// All public methods are safe for concurrent use.
98	type MemoryStore struct {
99		Metrics      map[string]MetricConfig
100		root         Level
101		nodeProvider NodeProvider
102	}
103
104	// Init initializes the metric store from configuration and starts background workers.
105	//
106	// This function must be called exactly once before any other metricstore operations.
107	// It performs the following initialization steps:
108	//  1. Validates and decodes the metric store configuration
109	//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
110	//  3. Loads metric configurations from all registered clusters
111	//  4. Restores checkpoints within the retention window
112	//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
113	//  6. Optionally subscribes to NATS for real-time metric ingestion
114	//
115	// Parameters:
116	//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig); may be nil to use defaults
117	//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
118	//   - wg: WaitGroup that will be incremented for each background goroutine started
119	//
120	// The function will call cclog.Fatal on critical errors during initialization.
121	// Use Shutdown() to cleanly stop all background workers started by Init().
122	//
123	// Note: Signal handling must be implemented by the caller. Call Shutdown() when
124	// receiving termination signals to ensure checkpoint data is persisted.
125	func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
126		startupTime := time.Now()
127
128		if rawConfig != nil {
129			config.Validate(configSchema, rawConfig)
130			dec := json.NewDecoder(bytes.NewReader(rawConfig))
131			dec.DisallowUnknownFields()
132			if err := dec.Decode(&Keys); err != nil {
133				cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
134			}
135		}
136
137		// Set NumWorkers from config or use default
138		if Keys.NumWorkers <= 0 {
139			Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
140		}
141		cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
142
143		// Pass the config.MetricStoreKeys
144		InitMetrics(metrics)
145
146		ms := GetMemoryStore()
147
148		d, err := time.ParseDuration(Keys.RetentionInMemory)
149		if err != nil {
150			cclog.Fatal(err)
151		}
152
153		restoreFrom := startupTime.Add(-d)
154		cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
155		files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
156		loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
157		if err != nil {
158			cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
159		} else {
160			cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
161		}
162
163		// Try to use less memory by forcing a GC run here and then
164		// lowering the target percentage. The default of 100 means
165		// that only once the ratio of new allocations execeds the
166		// previously active heap, a GC is triggered.
167		// Forcing a GC here will set the "previously active heap"
168		// to a minumum.
169		// runtime.GC()
170
171		ctx, shutdown := context.WithCancel(context.Background())
172
173		Retention(wg, ctx)
174		Checkpointing(wg, ctx)
175		CleanUp(wg, ctx)
176		WALStaging(wg, ctx)
177		MemoryUsageTracker(wg, ctx)
178
179		// Note: Signal handling has been removed from this function.
180		// The caller is responsible for handling shutdown signals and calling
181		// the shutdown() function when appropriate.
182		// Store the shutdown function for later use by Shutdown()
183		shutdownFuncMu.Lock()
184		shutdownFunc = shutdown
185		shutdownFuncMu.Unlock()
186
187		if Keys.Subscriptions != nil {
188			wg.Go(func() {
189				if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
190					cclog.Fatal(err)
191				}
192			})
193		}
194	}
195
196	// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
197	//
198	// This function must be called before GetMemoryStore() and can only be called once due to
199	// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
200	//
201	// Parameters:
202	//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
203	//
204	// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
205	//
206	// After this call, the global msInstance is ready for use via GetMemoryStore().
207	func InitMetrics(metrics map[string]MetricConfig) {
208		singleton.Do(func() {
209			offset := 0
210			for key, cfg := range metrics {
211				if cfg.Frequency == 0 {
212					panic("[METRICSTORE]> invalid frequency")
213				}
214
215				metrics[key] = MetricConfig{
216					Frequency:   cfg.Frequency,
217					Aggregation: cfg.Aggregation,
218					offset:      offset,
219				}
220				offset += 1
221			}
222
223			msInstance = &MemoryStore{
224				root: Level{
225					metrics:  make([]*buffer, len(metrics)),
226					children: make(map[string]*Level),
227				},
228				Metrics: metrics,
229			}
230		})
231	}
232
233	// GetMemoryStore returns the singleton MemoryStore instance.
234	//
235	// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
236	//
237	// This function is safe for concurrent use after initialization.
238	func GetMemoryStore() *MemoryStore {
239		if msInstance == nil {
240			cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
241		}
242
243		return msInstance
244	}
245
246	func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
247		if metric, ok := ms.Metrics[metricName]; ok {
248			return metric.Frequency, nil
249		}
250		return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
251	}
252
253	// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
254	// This must be called during initialization to provide job state information
255	// for selective buffer retention during Free operations.
256	// If not set, the Free function will fall back to freeing all buffers.
257	func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
258		ms.nodeProvider = provider
259	}
260
261	// Shutdown performs a graceful shutdown of the metric store.
262	//
263	// This function cancels all background goroutines started by Init() and writes
264	// a final checkpoint to disk before returning. It should be called when the
265	// application receives a termination signal.
266	//
267	// The function will:
268	//  1. Cancel the context to stop all background workers
269	//  2. Close the WAL messages channel if using WAL format
270	//  3. Write a final checkpoint to preserve in-memory data
271	//  4. Log any errors encountered during shutdown
272	//
273	// Note: This function blocks until the final checkpoint is written.
274	func Shutdown() {
275		totalStart := time.Now()
276
277		shutdownFuncMu.Lock()
278		defer shutdownFuncMu.Unlock()
279		if shutdownFunc != nil {
280			shutdownFunc()
281		}
282		cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))
283
284		if Keys.Checkpoints.FileFormat == "wal" {
285			// Signal producers to stop sending before closing channels,
286			// preventing send-on-closed-channel panics from in-flight NATS workers.
287			walShuttingDown.Store(true)
288			// Brief grace period for in-flight DecodeLine calls to complete.
289			time.Sleep(100 * time.Millisecond)
290
291			for _, ch := range walShardChs {
292				close(ch)
293			}
294			drainStart := time.Now()
295			WaitForWALStagingDrain()
296			cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
297		}
298
299		cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
300		checkpointStart := time.Now()
301		var files int
302		var err error
303
304		ms := GetMemoryStore()
305
306		lastCheckpointMu.Lock()
307		from := lastCheckpoint
308		lastCheckpointMu.Unlock()
309
310		if Keys.Checkpoints.FileFormat == "wal" {
311			// WAL files are deleted per-host inside ToCheckpointWAL workers.
312			files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
313		} else {
314			files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
315		}
316
317		if err != nil {
318			cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
319		}
320		cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
321	}
322
323	// Retention starts a background goroutine that periodically frees old metric data.
324	//
325	// This worker runs at half the retention interval and calls Free() to remove buffers
326	// older than the configured retention time. It respects the NodeProvider to preserve
327	// data for nodes with active jobs.
328	//
329	// Parameters:
330	//   - wg: WaitGroup to signal completion when context is cancelled
331	//   - ctx: Context for cancellation signal
332	//
333	// The goroutine exits when ctx is cancelled.
334	func Retention(wg *sync.WaitGroup, ctx context.Context) {
335		ms := GetMemoryStore()
336
337		wg.Go(func() {
338			d, err := time.ParseDuration(Keys.RetentionInMemory)
339			if err != nil {
340				cclog.Fatal(err)
341			}
342			if d <= 0 {
343				return
344			}
345
346			tickInterval := d / 2
347			if tickInterval <= 0 {
348				return
349			}
350			ticker := time.NewTicker(tickInterval)
351			defer ticker.Stop()
352
353			for {
354				select {
355				case <-ctx.Done():
356					return
357				case <-ticker.C:
358					state.mu.Lock()
359
360					t := time.Now().Add(-d)
361
362					state.lastRetentionTime = t.Unix()
363
364					cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
365
366					freed, err := Free(ms, t)
367					if err != nil {
368						cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
369					} else {
370						cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
371					}
372
373					state.mu.Unlock()
374
375					// Clean up the buffer pool
376					bufferPool.Clean(state.lastRetentionTime)
377				}
378			}
379		})
380	}
381
382	// MemoryUsageTracker starts a background goroutine that monitors memory usage.
383	//
384	// This worker checks actual process memory usage (via runtime.MemStats) periodically
385	// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
386	// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
387	// performance issues.
388	//
389	// The tracker logs both actual memory usage (heap allocated) and metric data size for
390	// visibility into memory overhead from Go runtime structures and allocations.
391	//
392	// Parameters:
393	//   - wg: WaitGroup to signal completion when context is cancelled
394	//   - ctx: Context for cancellation signal
395	//
396	// The goroutine exits when ctx is cancelled.
397	func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
398		ms := GetMemoryStore()
399
400		wg.Go(func() {
401			normalInterval := DefaultMemoryUsageTrackerInterval
402			fastInterval := 30 * time.Second
403
404			if normalInterval <= 0 {
405				return
406			}
407
408			ticker := time.NewTicker(normalInterval)
409			defer ticker.Stop()
410			currentInterval := normalInterval
411
412			for {
413				select {
414				case <-ctx.Done():
415					return
416				case <-ticker.C:
417					var mem runtime.MemStats
418					runtime.ReadMemStats(&mem)
419					actualMemoryGB := float64(mem.Alloc) / 1e9
420					metricDataGB := ms.SizeInGB()
421					cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
422
423					freedExcluded := 0
424					freedEmergency := 0
425					var err error
426
427					state.mu.RLock()
428					lastRetention := state.lastRetentionTime
429					selectorsExcluded := state.selectorsExcluded
430					state.mu.RUnlock()
431
432					if lastRetention != 0 && selectorsExcluded {
433						freedExcluded, err = ms.Free(nil, lastRetention)
434						if err != nil {
435							cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
436						}
437
438						if freedExcluded > 0 {
439							debug.FreeOSMemory()
440							cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
441						}
442					}
443
444					runtime.ReadMemStats(&mem)
445					actualMemoryGB = float64(mem.Alloc) / 1e9
446
447					if actualMemoryGB > float64(Keys.MemoryCap) {
448						cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)
449
450						// Use progressive time-based Free with increasing threshold
451						// instead of ForceFree loop — fewer tree traversals, more effective
452						d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
453						if parseErr != nil {
454							cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
455						} else {
456							thresholds := []float64{0.75, 0.5, 0.25}
457							for _, fraction := range thresholds {
458								threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
459								freed, freeErr := ms.Free(nil, threshold.Unix())
460								if freeErr != nil {
461									cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
462								}
463								freedEmergency += freed
464
465								bufferPool.Clear()
466								runtime.GC()
467								runtime.ReadMemStats(&mem)
468								actualMemoryGB = float64(mem.Alloc) / 1e9
469
470								if actualMemoryGB < float64(Keys.MemoryCap) {
471									break
472								}
473							}
474						}
475
476						bufferPool.Clear()
477						debug.FreeOSMemory()
478
479						runtime.ReadMemStats(&mem)
480						actualMemoryGB = float64(mem.Alloc) / 1e9
481
482						if actualMemoryGB >= float64(Keys.MemoryCap) {
483							cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
484						} else {
485							cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
486						}
487					}
488
489					// Adaptive ticker: check more frequently when memory is high
490					memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
491					if memoryRatio > 0.8 && currentInterval != fastInterval {
492						ticker.Reset(fastInterval)
493						currentInterval = fastInterval
494						cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
495					} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
496						ticker.Reset(normalInterval)
497						currentInterval = normalInterval
498						cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
499					}
500				}
501			}
502		})
503	}
504
505	// Free removes metric data older than the given time while preserving data for active nodes.
506	//
507	// This function implements intelligent retention by consulting the NodeProvider (if configured)
508	// to determine which nodes are currently in use by running jobs. Data for these nodes is
509	// preserved even if older than the retention time.
510	//
511	// Parameters:
512	//   - ms: The MemoryStore instance
513	//   - t: Time threshold - buffers with data older than this will be freed
514	//
515	// Returns:
516	//   - Number of buffers freed
517	//   - Error if NodeProvider query fails
518	//
519	// Behavior:
520	//   - If no NodeProvider is set: frees all buffers older than t
521	//   - If NodeProvider returns empty map: frees all buffers older than t
522	//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
523	func Free(ms *MemoryStore, t time.Time) (int, error) {
524		// If no NodeProvider is configured, free all buffers older than t
525		if ms.nodeProvider == nil {
526			return ms.Free(nil, t.Unix())
527		}
528
529		excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
530		if err != nil {
531			return 0, err
532		}
533
534		switch lenMap := len(excludeSelectors); lenMap {
535
536		// If the length of the map returned by GetUsedNodes() is 0,
537		// then use default Free method with nil selector
538		case 0:
539			state.selectorsExcluded = false
540			return ms.Free(nil, t.Unix())
541
542		// Else formulate selectors, exclude those from the map
543		// and free the rest of the selectors
544		default:
545			state.selectorsExcluded = true
546			selectors := GetSelectors(ms, excludeSelectors)
547			return FreeSelected(ms, selectors, t)
548		}
549	}
550
551	// FreeSelected frees buffers for specific selectors while preserving others.
552	//
553	// This function is used when we want to retain some specific nodes beyond the retention time.
554	// It iterates through the provided selectors and frees their associated buffers.
555	//
556	// Parameters:
557	//   - ms: The MemoryStore instance
558	//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
559	//   - t: Time threshold for freeing buffers
560	//
561	// Returns the total number of buffers freed and any error encountered.
562	func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
563		freed := 0
564
565		for _, selector := range selectors {
566
567			freedBuffers, err := ms.Free(selector, t.Unix())
568			if err != nil {
569				cclog.Errorf("error while freeing selected buffers: %#v", err)
570			}
571			freed += freedBuffers
572
573		}
574
575		return freed, nil
576	}
577
578	// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
579	//
580	// This function generates a list of selectors whose buffers should be freed by excluding
581	// selectors that correspond to nodes currently in use by running jobs.
582	//
583	// Parameters:
584	//   - ms: The MemoryStore instance
585	//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
586	//
587	// Returns a list of selectors ([]string paths) that can be safely freed.
588	//
589	// Example:
590	//
591	//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
592	//	and excludeSelectors contains {"emmy": ["node001"]},
593	//	then only [["emmy", "node002"]] is returned.
594	func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
595		allSelectors := ms.GetPaths(2)
596
597		filteredSelectors := make([][]string, 0, len(allSelectors))
598
599		for _, path := range allSelectors {
600			if len(path) < 2 {
601				continue
602			}
603
604			key := path[0]   // The "Key" (Level 1)
605			value := path[1] // The "Value" (Level 2)
606
607			exclude := false
608
609			// Check if the key exists in our exclusion map
610			if excludedValues, exists := excludeSelectors[key]; exists {
611				// The key exists, now check if the specific value is in the exclusion list
612				if slices.Contains(excludedValues, value) {
613					exclude = true
614				}
615			}
616
617			if !exclude {
618				filteredSelectors = append(filteredSelectors, path)
619			}
620		}
621
622		return filteredSelectors
623	}
624
625	// GetPaths returns a list of lists (paths) to the specified depth.
626	func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
627		var results [][]string
628
629		// Start recursion. Initial path is empty.
630		// We treat Root as depth 0.
631		ms.root.collectPaths(0, targetDepth, []string{}, &results)
632
633		return results
634	}
635
636	// Write all values in `metrics` to the level specified by `selector` for time `ts`.
637	// Look at `findLevelOrCreate` for how selectors work.
638	func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
639		var ok bool
640		for i, metric := range metrics {
641			if metric.MetricConfig.Frequency == 0 {
642				metric.MetricConfig, ok = m.Metrics[metric.Name]
643				if !ok {
644					cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
645					metric.MetricConfig.Frequency = 0
646				}
647				metrics[i] = metric
648			}
649		}
650
651		return m.WriteToLevel(&m.root, selector, ts, metrics)
652	}
653
654	func (m *MemoryStore) GetLevel(selector []string) *Level {
655		return m.root.findLevelOrCreate(selector, len(m.Metrics))
656	}
657
658	// WriteToLevel assumes that `minfo` in `metrics` is filled in
659	func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
660		l = l.findLevelOrCreate(selector, len(m.Metrics))
661		l.lock.Lock()
662		defer l.lock.Unlock()
663
664		for _, metric := range metrics {
665			if metric.MetricConfig.Frequency == 0 {
666				continue
667			}
668
669			b := l.metrics[metric.MetricConfig.offset]
670			if b == nil {
671				// First write to this metric and level
672				b = newBuffer(ts, metric.MetricConfig.Frequency)
673				l.metrics[metric.MetricConfig.offset] = b
674			}
675
676			nb, err := b.write(ts, metric.Value)
677			if err != nil {
678				return err
679			}
680
681			// Last write created a new buffer...
682			if b != nb {
683				l.metrics[metric.MetricConfig.offset] = nb
684			}
685		}
686		return nil
687	}
688
689	// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
690	// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
691	// The second and third return value are the actual from/to for the data. Those can be different from
692	// the range asked for if no data was available.
693	func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
694		if from > to {
695			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
696		}
697
698		minfo, ok := m.Metrics[metric]
699		if !ok {
700			return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
701		}
702
703		n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
704
705		err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
706			cdata, cfrom, cto, err := b.read(from, to, data)
707			if err != nil {
708				return err
709			}
710
711			if n == 0 {
712				from, to = cfrom, cto
713			} else if from != cfrom || to != cto || len(data) != len(cdata) {
714				missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
715				if missingfront != 0 {
716					return ErrDataDoesNotAlignMissingFront
717				}
718
719				newlen := len(cdata) - missingback
720				if newlen < 1 {
721					return ErrDataDoesNotAlignMissingBack
722				}
723				cdata = cdata[0:newlen]
724				if len(cdata) != len(data) {
725					return ErrDataDoesNotAlignDataLenMismatch
726				}
727
728				from, to = cfrom, cto
729			}
730
731			data = cdata
732			n += 1
733			return nil
734		})
735
736		if err != nil {
737			return nil, 0, 0, 0, err
738		} else if n == 0 {
739			return nil, 0, 0, 0, ErrNoHostOrMetric
740		} else if n > 1 {
741			if minfo.Aggregation == AvgAggregation {
742				normalize := 1. / schema.Float(n)
743				for i := 0; i < len(data); i++ {
744					data[i] *= normalize
745				}
746			} else if minfo.Aggregation != SumAggregation {
747				return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
748			}
749		}
750
751		data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
752		if err != nil {
753			return nil, 0, 0, 0, err
754		}
755
756		return data, from, to, resolution, nil
757	}
758
759	// Free releases all buffers for the selected level and all its children that
760	// contain only values older than `t`.
761	func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
762		return m.GetLevel(selector).free(t)
763	}
764
765	// ForceFree unconditionally removes the oldest buffer from each metric chain.
766	func (m *MemoryStore) ForceFree() (int, error) {
767		return m.GetLevel(nil).forceFree()
768	}
769
770	func (m *MemoryStore) FreeAll() error {
771		m.root.lock.Lock()
772		defer m.root.lock.Unlock()
773
774		for k := range m.root.children {
775			delete(m.root.children, k)
776		}
777
778		return nil
779	}
780
781	func (m *MemoryStore) SizeInBytes() int64 {
782		return m.root.sizeInBytes()
783	}
784
785	func (m *MemoryStore) SizeInGB() float64 {
786		return float64(m.root.sizeInBytes()) / 1e9
787	}
788
789	// ListChildren , given a selector, returns a list of all children of the level
790	// selected.
791	func (m *MemoryStore) ListChildren(selector []string) []string {
792		lvl := &m.root
793		for lvl != nil && len(selector) != 0 {
794			lvl.lock.RLock()
795			next := lvl.children[selector[0]]
796			lvl.lock.RUnlock()
797			lvl = next
798			selector = selector[1:]
799		}
800
801		if lvl == nil {
802			return nil
803		}
804
805		lvl.lock.RLock()
806		defer lvl.lock.RUnlock()
807
808		children := make([]string, 0, len(lvl.children))
809		for child := range lvl.children {
810			children = append(children, child)
811		}
812
813		return children
814	}
815

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides an efficient in-memory time-series metric storage system
// with support for hierarchical data organization, checkpointing, and archiving.
//
// The package organizes metrics in a tree structure (cluster → host → component) and
// provides concurrent read/write access to metric data with configurable aggregation strategies.
// Background goroutines handle periodic checkpointing (JSON or WAL/binary format), archiving old data,
// and enforcing retention policies.
//
// Key features:
//   - In-memory metric storage with configurable retention
//   - Hierarchical data organization (selectors)
//   - Concurrent checkpoint/archive workers
//   - Support for sum and average aggregation
//   - NATS integration for metric ingestion via InfluxDB line protocol
package metricstore

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"runtime"
	"runtime/debug"
	"slices"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/config"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/resampler"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-lib/v2/util"
)

// GlobalState holds the global state for the metric store with thread-safe access.
type GlobalState struct {
	mu                sync.RWMutex
	lastRetentionTime int64
	selectorsExcluded bool
}

var (
	singleton  sync.Once
	msInstance *MemoryStore
	// shutdownFunc stores the context cancellation function created in Init
	// and is called during Shutdown to cancel all background goroutines
	shutdownFunc   context.CancelFunc
	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
	// Create a global instance
	state = &GlobalState{}
)

// NodeProvider provides information about nodes currently in use by running jobs.
//
// This interface allows metricstore to query job information without directly
// depending on the repository package, breaking the import cycle.
//
// Implementations should return nodes that are actively processing jobs started
// before the given timestamp. These nodes will be excluded from retention-based
// garbage collection to prevent data loss for jobs that are still running or
// recently completed.
type NodeProvider interface {
	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
	// that are currently in use by jobs that started before the given timestamp.
	//
	// Parameters:
	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
	//
	// Returns:
	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
	//   - Error if the query fails
	GetUsedNodes(ts int64) (map[string][]string, error)
}

// Metric represents a single metric data point to be written to the store.
type Metric struct {
	Name  string
	Value schema.Float
	// MetricConfig contains frequency and aggregation settings for this metric.
	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
	MetricConfig MetricConfig
}

// MemoryStore is the main in-memory time-series metric storage implementation.
//
// It organizes metrics in a hierarchical tree structure where each level represents
// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
// store multiple metrics as time-series buffers.
//
// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
// All public methods are safe for concurrent use.
type MemoryStore struct {
	Metrics      map[string]MetricConfig
	root         Level
	nodeProvider NodeProvider
}

// Init initializes the metric store from configuration and starts background workers.
//
// This function must be called exactly once before any other metricstore operations.
// It performs the following initialization steps:
//  1. Validates and decodes the metric store configuration
//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
//  3. Loads metric configurations from all registered clusters
//  4. Restores checkpoints within the retention window
//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
//  6. Optionally subscribes to NATS for real-time metric ingestion
//
// Parameters:
//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig); may be nil to use defaults
//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
//   - wg: WaitGroup that will be incremented for each background goroutine started
//
// The function will call cclog.Fatal on critical errors during initialization.
// Use Shutdown() to cleanly stop all background workers started by Init().
//
// Note: Signal handling must be implemented by the caller. Call Shutdown() when
// receiving termination signals to ensure checkpoint data is persisted.
func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
	startupTime := time.Now()

	if rawConfig != nil {
		config.Validate(configSchema, rawConfig)
		dec := json.NewDecoder(bytes.NewReader(rawConfig))
		dec.DisallowUnknownFields()
		if err := dec.Decode(&Keys); err != nil {
			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
		}
	}

	// Set NumWorkers from config or use default
	if Keys.NumWorkers <= 0 {
		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
	}
	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)

	// Pass the config.MetricStoreKeys
	InitMetrics(metrics)

	ms := GetMemoryStore()

	d, err := time.ParseDuration(Keys.RetentionInMemory)
	if err != nil {
		cclog.Fatal(err)
	}

	restoreFrom := startupTime.Add(-d)
	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
	if err != nil {
		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
	} else {
		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
	}

	// Try to use less memory by forcing a GC run here and then
	// lowering the target percentage. The default of 100 means
	// that only once the ratio of new allocations execeds the
	// previously active heap, a GC is triggered.
	// Forcing a GC here will set the "previously active heap"
	// to a minumum.
	// runtime.GC()

	ctx, shutdown := context.WithCancel(context.Background())

	Retention(wg, ctx)
	Checkpointing(wg, ctx)
	CleanUp(wg, ctx)
	WALStaging(wg, ctx)
	MemoryUsageTracker(wg, ctx)

	// Note: Signal handling has been removed from this function.
	// The caller is responsible for handling shutdown signals and calling
	// the shutdown() function when appropriate.
	// Store the shutdown function for later use by Shutdown()
	shutdownFuncMu.Lock()
	shutdownFunc = shutdown
	shutdownFuncMu.Unlock()

	if Keys.Subscriptions != nil {
		wg.Go(func() {
			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
				cclog.Fatal(err)
			}
		})
	}
}

// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
//
// This function must be called before GetMemoryStore() and can only be called once due to
// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
//
// Parameters:
//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
//
// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
//
// After this call, the global msInstance is ready for use via GetMemoryStore().
func InitMetrics(metrics map[string]MetricConfig) {
	singleton.Do(func() {
		offset := 0
		for key, cfg := range metrics {
			if cfg.Frequency == 0 {
				panic("[METRICSTORE]> invalid frequency")
			}

			metrics[key] = MetricConfig{
				Frequency:   cfg.Frequency,
				Aggregation: cfg.Aggregation,
				offset:      offset,
			}
			offset += 1
		}

		msInstance = &MemoryStore{
			root: Level{
				metrics:  make([]*buffer, len(metrics)),
				children: make(map[string]*Level),
			},
			Metrics: metrics,
		}
	})
}

// GetMemoryStore returns the singleton MemoryStore instance.
//
// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
//
// This function is safe for concurrent use after initialization.
func GetMemoryStore() *MemoryStore {
	if msInstance == nil {
		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
	}

	return msInstance
}

func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
	if metric, ok := ms.Metrics[metricName]; ok {
		return metric.Frequency, nil
	}
	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
}

// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
// This must be called during initialization to provide job state information
// for selective buffer retention during Free operations.
// If not set, the Free function will fall back to freeing all buffers.
func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
	ms.nodeProvider = provider
}

// Shutdown performs a graceful shutdown of the metric store.
//
// This function cancels all background goroutines started by Init() and writes
// a final checkpoint to disk before returning. It should be called when the
// application receives a termination signal.
//
// The function will:
//  1. Cancel the context to stop all background workers
//  2. Close the WAL messages channel if using WAL format
//  3. Write a final checkpoint to preserve in-memory data
//  4. Log any errors encountered during shutdown
//
// Note: This function blocks until the final checkpoint is written.
func Shutdown() {
	totalStart := time.Now()

	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}
	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))

	if Keys.Checkpoints.FileFormat == "wal" {
		// Signal producers to stop sending before closing channels,
		// preventing send-on-closed-channel panics from in-flight NATS workers.
		walShuttingDown.Store(true)
		// Brief grace period for in-flight DecodeLine calls to complete.
		time.Sleep(100 * time.Millisecond)

		for _, ch := range walShardChs {
			close(ch)
		}
		drainStart := time.Now()
		WaitForWALStagingDrain()
		cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
	}

	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
	checkpointStart := time.Now()
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		// WAL files are deleted per-host inside ToCheckpointWAL workers.
		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
}

// Retention starts a background goroutine that periodically frees old metric data.
//
// This worker runs at half the retention interval and calls Free() to remove buffers
// older than the configured retention time. It respects the NodeProvider to preserve
// data for nodes with active jobs.
//
// Parameters:
//   - wg: WaitGroup to signal completion when context is cancelled
//   - ctx: Context for cancellation signal
//
// The goroutine exits when ctx is cancelled.
func Retention(wg *sync.WaitGroup, ctx context.Context) {
	ms := GetMemoryStore()

	wg.Go(func() {
		d, err := time.ParseDuration(Keys.RetentionInMemory)
		if err != nil {
			cclog.Fatal(err)
		}
		if d <= 0 {
			return
		}

		tickInterval := d / 2
		if tickInterval <= 0 {
			return
		}
		ticker := time.NewTicker(tickInterval)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				state.mu.Lock()

				t := time.Now().Add(-d)

				state.lastRetentionTime = t.Unix()

				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))

				freed, err := Free(ms, t)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
				} else {
					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
				}

				state.mu.Unlock()

				// Clean up the buffer pool
				bufferPool.Clean(state.lastRetentionTime)
			}
		}
	})
}

// MemoryUsageTracker starts a background goroutine that monitors memory usage.
//
// This worker checks actual process memory usage (via runtime.MemStats) periodically
// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
// performance issues.
//
// The tracker logs both actual memory usage (heap allocated) and metric data size for
// visibility into memory overhead from Go runtime structures and allocations.
//
// Parameters:
//   - wg: WaitGroup to signal completion when context is cancelled
//   - ctx: Context for cancellation signal
//
// The goroutine exits when ctx is cancelled.
func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
	ms := GetMemoryStore()

	wg.Go(func() {
		normalInterval := DefaultMemoryUsageTrackerInterval
		fastInterval := 30 * time.Second

		if normalInterval <= 0 {
			return
		}

		ticker := time.NewTicker(normalInterval)
		defer ticker.Stop()
		currentInterval := normalInterval

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				var mem runtime.MemStats
				runtime.ReadMemStats(&mem)
				actualMemoryGB := float64(mem.Alloc) / 1e9
				metricDataGB := ms.SizeInGB()
				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)

				freedExcluded := 0
				freedEmergency := 0
				var err error

				state.mu.RLock()
				lastRetention := state.lastRetentionTime
				selectorsExcluded := state.selectorsExcluded
				state.mu.RUnlock()

				if lastRetention != 0 && selectorsExcluded {
					freedExcluded, err = ms.Free(nil, lastRetention)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
					}

					if freedExcluded > 0 {
						debug.FreeOSMemory()
						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
					}
				}

				runtime.ReadMemStats(&mem)
				actualMemoryGB = float64(mem.Alloc) / 1e9

				if actualMemoryGB > float64(Keys.MemoryCap) {
					cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)

					// Use progressive time-based Free with increasing threshold
					// instead of ForceFree loop — fewer tree traversals, more effective
					d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
					if parseErr != nil {
						cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
					} else {
						thresholds := []float64{0.75, 0.5, 0.25}
						for _, fraction := range thresholds {
							threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
							freed, freeErr := ms.Free(nil, threshold.Unix())
							if freeErr != nil {
								cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
							}
							freedEmergency += freed

							bufferPool.Clear()
							runtime.GC()
							runtime.ReadMemStats(&mem)
							actualMemoryGB = float64(mem.Alloc) / 1e9

							if actualMemoryGB < float64(Keys.MemoryCap) {
								break
							}
						}
					}

					bufferPool.Clear()
					debug.FreeOSMemory()

					runtime.ReadMemStats(&mem)
					actualMemoryGB = float64(mem.Alloc) / 1e9

					if actualMemoryGB >= float64(Keys.MemoryCap) {
						cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
					} else {
						cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
					}
				}

				// Adaptive ticker: check more frequently when memory is high
				memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
				if memoryRatio > 0.8 && currentInterval != fastInterval {
					ticker.Reset(fastInterval)
					currentInterval = fastInterval
					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
				} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
					ticker.Reset(normalInterval)
					currentInterval = normalInterval
					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
				}
			}
		}
	})
}

// Free removes metric data older than the given time while preserving data for active nodes.
//
// This function implements intelligent retention by consulting the NodeProvider (if configured)
// to determine which nodes are currently in use by running jobs. Data for these nodes is
// preserved even if older than the retention time.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - t: Time threshold - buffers with data older than this will be freed
//
// Returns:
//   - Number of buffers freed
//   - Error if NodeProvider query fails
//
// Behavior:
//   - If no NodeProvider is set: frees all buffers older than t
//   - If NodeProvider returns empty map: frees all buffers older than t
//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
func Free(ms *MemoryStore, t time.Time) (int, error) {
	// If no NodeProvider is configured, free all buffers older than t
	if ms.nodeProvider == nil {
		return ms.Free(nil, t.Unix())
	}

	excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
	if err != nil {
		return 0, err
	}

	switch lenMap := len(excludeSelectors); lenMap {

	// If the length of the map returned by GetUsedNodes() is 0,
	// then use default Free method with nil selector
	case 0:
		state.selectorsExcluded = false
		return ms.Free(nil, t.Unix())

	// Else formulate selectors, exclude those from the map
	// and free the rest of the selectors
	default:
		state.selectorsExcluded = true
		selectors := GetSelectors(ms, excludeSelectors)
		return FreeSelected(ms, selectors, t)
	}
}

// FreeSelected frees buffers for specific selectors while preserving others.
//
// This function is used when we want to retain some specific nodes beyond the retention time.
// It iterates through the provided selectors and frees their associated buffers.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
//   - t: Time threshold for freeing buffers
//
// Returns the total number of buffers freed and any error encountered.
func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
	freed := 0

	for _, selector := range selectors {

		freedBuffers, err := ms.Free(selector, t.Unix())
		if err != nil {
			cclog.Errorf("error while freeing selected buffers: %#v", err)
		}
		freed += freedBuffers

	}

	return freed, nil
}

// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
//
// This function generates a list of selectors whose buffers should be freed by excluding
// selectors that correspond to nodes currently in use by running jobs.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
//
// Returns a list of selectors ([]string paths) that can be safely freed.
//
// Example:
//
//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
//	and excludeSelectors contains {"emmy": ["node001"]},
//	then only [["emmy", "node002"]] is returned.
func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
	allSelectors := ms.GetPaths(2)

	filteredSelectors := make([][]string, 0, len(allSelectors))

	for _, path := range allSelectors {
		if len(path) < 2 {
			continue
		}

		key := path[0]   // The "Key" (Level 1)
		value := path[1] // The "Value" (Level 2)

		exclude := false

		// Check if the key exists in our exclusion map
		if excludedValues, exists := excludeSelectors[key]; exists {
			// The key exists, now check if the specific value is in the exclusion list
			if slices.Contains(excludedValues, value) {
				exclude = true
			}
		}

		if !exclude {
			filteredSelectors = append(filteredSelectors, path)
		}
	}

	return filteredSelectors
}

// GetPaths returns a list of lists (paths) to the specified depth.
func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
	var results [][]string

	// Start recursion. Initial path is empty.
	// We treat Root as depth 0.
	ms.root.collectPaths(0, targetDepth, []string{}, &results)

	return results
}

// Write all values in `metrics` to the level specified by `selector` for time `ts`.
// Look at `findLevelOrCreate` for how selectors work.
func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
	var ok bool
	for i, metric := range metrics {
		if metric.MetricConfig.Frequency == 0 {
			metric.MetricConfig, ok = m.Metrics[metric.Name]
			if !ok {
				cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
				metric.MetricConfig.Frequency = 0
			}
			metrics[i] = metric
		}
	}

	return m.WriteToLevel(&m.root, selector, ts, metrics)
}

func (m *MemoryStore) GetLevel(selector []string) *Level {
	return m.root.findLevelOrCreate(selector, len(m.Metrics))
}

// WriteToLevel assumes that `minfo` in `metrics` is filled in
func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
	l = l.findLevelOrCreate(selector, len(m.Metrics))
	l.lock.Lock()
	defer l.lock.Unlock()

	for _, metric := range metrics {
		if metric.MetricConfig.Frequency == 0 {
			continue
		}

		b := l.metrics[metric.MetricConfig.offset]
		if b == nil {
			// First write to this metric and level
			b = newBuffer(ts, metric.MetricConfig.Frequency)
			l.metrics[metric.MetricConfig.offset] = b
		}

		nb, err := b.write(ts, metric.Value)
		if err != nil {
			return err
		}

		// Last write created a new buffer...
		if b != nb {
			l.metrics[metric.MetricConfig.offset] = nb
		}
	}
	return nil
}

// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
// The second and third return value are the actual from/to for the data. Those can be different from
// the range asked for if no data was available.
func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
	if from > to {
		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
	}

	minfo, ok := m.Metrics[metric]
	if !ok {
		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
	}

	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)

	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
		cdata, cfrom, cto, err := b.read(from, to, data)
		if err != nil {
			return err
		}

		if n == 0 {
			from, to = cfrom, cto
		} else if from != cfrom || to != cto || len(data) != len(cdata) {
			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
			if missingfront != 0 {
				return ErrDataDoesNotAlignMissingFront
			}

			newlen := len(cdata) - missingback
			if newlen < 1 {
				return ErrDataDoesNotAlignMissingBack
			}
			cdata = cdata[0:newlen]
			if len(cdata) != len(data) {
				return ErrDataDoesNotAlignDataLenMismatch
			}

			from, to = cfrom, cto
		}

		data = cdata
		n += 1
		return nil
	})

	if err != nil {
		return nil, 0, 0, 0, err
	} else if n == 0 {
		return nil, 0, 0, 0, ErrNoHostOrMetric
	} else if n > 1 {
		if minfo.Aggregation == AvgAggregation {
			normalize := 1. / schema.Float(n)
			for i := 0; i < len(data); i++ {
				data[i] *= normalize
			}
		} else if minfo.Aggregation != SumAggregation {
			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
		}
	}

	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
	if err != nil {
		return nil, 0, 0, 0, err
	}

	return data, from, to, resolution, nil
}

// Free releases all buffers for the selected level and all its children that
// contain only values older than `t`.
func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
	return m.GetLevel(selector).free(t)
}

// ForceFree unconditionally removes the oldest buffer from each metric chain.
func (m *MemoryStore) ForceFree() (int, error) {
	return m.GetLevel(nil).forceFree()
}

func (m *MemoryStore) FreeAll() error {
	m.root.lock.Lock()
	defer m.root.lock.Unlock()

	for k := range m.root.children {
		delete(m.root.children, k)
	}

	return nil
}

func (m *MemoryStore) SizeInBytes() int64 {
	return m.root.sizeInBytes()
}

func (m *MemoryStore) SizeInGB() float64 {
	return float64(m.root.sizeInBytes()) / 1e9
}

// ListChildren , given a selector, returns a list of all children of the level
// selected.
func (m *MemoryStore) ListChildren(selector []string) []string {
	lvl := &m.root
	for lvl != nil && len(selector) != 0 {
		lvl.lock.RLock()
		next := lvl.children[selector[0]]
		lvl.lock.RUnlock()
		lvl = next
		selector = selector[1:]
	}

	if lvl == nil {
		return nil
	}

	lvl.lock.RLock()
	defer lvl.lock.RUnlock()

	children := make([]string, 0, len(lvl.children))
	for child := range lvl.children {
		children = append(children, child)
	}

	return children
}


Bash

grep -rn "MaxWALSize\|FileFormat\|CheckpointInterval\|walDropped\|RotateWALFilesAfterShutdown\|WaitForWALStagingDrain\|walShuttingDown" pkg/metricstore/ --include=*.go | grep -v "_test.go"


This application uses a files based checkpointing mechanism using a WAL algorithm for crash recovery. Analyse when the wal files are reset. Currently the wal files are not reset and continously grow a…


Analyze WAL file reset mechanism in checkpoint system


(eval):1: no matches found: --include=*.go


grep -rn "MaxWALSize\|FileFormat\|CheckpointInterval\|MaxWALFiles\|RootDir\|archived" pkg/metricstore/config.go pkg/metricstore/buffer.go 2>/dev/null; echo "---- archived set ----"; grep -rn "\.archived" pkg/metricstore/*.go | grep -v _test


pkg/metricstore/buffer.go:165://   - archived:  True if data has been persisted to disk archive
pkg/metricstore/buffer.go:176:	archived  bool
pkg/metricstore/buffer.go:187:	b.archived = false
pkg/metricstore/config.go:17://	│  ├─ FileFormat: "json" or "wal" (default: "wal")
pkg/metricstore/config.go:18://	│  └─ RootDir: Checkpoint storage path
pkg/metricstore/config.go:20://	│  ├─ RootDir: Archive storage path (archive mode only)
pkg/metricstore/config.go:62://   - FileFormat:  "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal"
pkg/metricstore/config.go:63://   - RootDir:     Filesystem path for checkpoint files (created if missing)
pkg/metricstore/config.go:64://   - MaxWALSize:  Maximum size in bytes for a single host's WAL file; 0 = unlimited (default).
pkg/metricstore/config.go:67:	FileFormat string `json:"file-format"`
pkg/metricstore/config.go:68:	RootDir    string `json:"directory"`
pkg/metricstore/config.go:69:	MaxWALSize int64  `json:"max-wal-size,omitempty"`
pkg/metricstore/config.go:84:// Data older than RetentionInMemory is archived to disk or deleted.
pkg/metricstore/config.go:88://   - RootDir: Filesystem path for archived data (used in "archive" mode)
pkg/metricstore/config.go:89://   - Mode:    "delete" (discard old data) or "archive" (write to RootDir)
pkg/metricstore/config.go:91:	RootDir string `json:"directory"`
pkg/metricstore/config.go:129:	CheckpointInterval string         `json:"checkpoint-interval,omitempty"`
pkg/metricstore/config.go:143:		FileFormat: "wal",
pkg/metricstore/config.go:144:		RootDir:    "./var/checkpoints",
---- archived set ----
pkg/metricstore/buffer.go:187:	b.archived = false
pkg/metricstore/checkpoint.go:289:			if !b.archived {
pkg/metricstore/debug.go:26:	if b.archived {


160	//   - prev:      Link to older buffer in the chain (nil if this is oldest)
161	//   - next:      Link to newer buffer in the chain (nil if this is newest/head)
162	//   - data:      Time-series values (schema.Float supports NaN for missing data)
163	//   - frequency: Measurement interval in seconds
164	//   - start:     Start timestamp (adjusted by -frequency/2 for alignment)
165	//   - archived:  True if data has been persisted to disk archive
166	//   - closed:    True if buffer is no longer accepting writes
167	//
168	// Index calculation: index = (timestamp - start) / frequency
169	// Actual data timestamp: start + (frequency / 2) + (index * frequency)
170	type buffer struct {
171		prev      *buffer
172		next      *buffer
173		data      []schema.Float
174		frequency int64
175		start     int64
176		archived  bool
177		closed    bool
178		lastUsed  int64
179	}
180
181	func newBuffer(ts, freq int64) *buffer {
182		b := bufferPool.Get()
183		b.frequency = freq
184		b.start = ts - (freq / 2)
185		b.prev = nil
186		b.next = nil
187		b.archived = false
188		b.closed = false
189		b.data = b.data[:0]
190		return b
191	}
192
193	// write appends a timestamped value to the buffer chain.
194	//
195	// Returns the head buffer (which may be newly created if capacity was reached).
196	// Timestamps older than the buffer's start are rejected. If the calculated index
197	// exceeds capacity, a new buffer is allocated and linked as the new head.
198	//
199	// Missing timestamps are automatically filled with NaN values to maintain alignment.
200	// Overwrites are allowed if the index is already within the existing data slice.
201	//
202	// Parameters:
203	//   - ts:    Unix timestamp in seconds
204	//   - value: Metric value (can be schema.NaN for missing data)
205	//
206	// Returns:
207	//   - *buffer: The new head buffer (same as b if no new buffer created)
208	//   - error:   Non-nil if timestamp is before buffer start
209	func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
210		if ts < b.start {
211			return nil, errors.New("[METRICSTORE]> cannot write value to buffer from past")
212		}
213
214		// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
215		idx := int((ts - b.start) / b.frequency)
216		if idx >= cap(b.data) {
217			newbuf := newBuffer(ts, b.frequency)
218			newbuf.prev = b
219			b.next = newbuf
220			b = newbuf
221			idx = 0
222		}
223
224		// Overwriting value or writing value from past
225		if idx < len(b.data) {
226			b.data[idx] = value
227			return b, nil
228		}
229
230		// Fill up unwritten slots with NaN
231		for i := len(b.data); i < idx; i++ {
232			b.data = append(b.data, schema.NaN)
233		}
234
235		b.data = append(b.data, value)
236		return b, nil
237	}
238
239	func (b *buffer) end() int64 {
240		return b.firstWrite() + int64(len(b.data))*b.frequency
241	}
242
243	func (b *buffer) firstWrite() int64 {
244		return b.start + (b.frequency / 2)
245	}
246
247	// read retrieves time-series data from the buffer chain for the specified time range.
248	//
249	// Traverses the buffer chain backwards (via prev links) if 'from' precedes the current
250	// buffer's start. Missing data points are represented as NaN. Values are accumulated
251	// into the provided 'data' slice (using +=, so caller must zero-initialize if needed).
252	//
253	// The function adjusts the actual time range returned if data is unavailable at the
254	// boundaries (returned via adjusted from/to timestamps).
255	//
256	// Parameters:
257	//   - from: Start timestamp (Unix seconds)
258	//   - to:   End timestamp (Unix seconds, exclusive)
259	//   - data: Pre-allocated slice to accumulate results (must be large enough)
260	//
261	// Returns:
262	//   - []schema.Float: Slice of data (may be shorter than input 'data' slice)
263	//   - int64:          Actual start timestamp with available data
264	//   - int64:          Actual end timestamp (exclusive)
265	//   - error:          Non-nil on failure
266	//
267	// Panics if 'data' slice is too small to hold all values in [from, to).
268	func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) {
269		// Walk back to the buffer that covers 'from', adjusting if we hit the oldest.
270		for from < b.firstWrite() {
271			if b.prev == nil {
272				from = b.firstWrite()
273				break
274			}
275			b = b.prev
276		}
277
278		i := 0
279		t := from

//   - prev:      Link to older buffer in the chain (nil if this is oldest)
//   - next:      Link to newer buffer in the chain (nil if this is newest/head)
//   - data:      Time-series values (schema.Float supports NaN for missing data)
//   - frequency: Measurement interval in seconds
//   - start:     Start timestamp (adjusted by -frequency/2 for alignment)
//   - archived:  True if data has been persisted to disk archive
//   - closed:    True if buffer is no longer accepting writes
//
// Index calculation: index = (timestamp - start) / frequency
// Actual data timestamp: start + (frequency / 2) + (index * frequency)
type buffer struct {
	prev      *buffer
	next      *buffer
	data      []schema.Float
	frequency int64
	start     int64
	archived  bool
	closed    bool
	lastUsed  int64
}

func newBuffer(ts, freq int64) *buffer {
	b := bufferPool.Get()
	b.frequency = freq
	b.start = ts - (freq / 2)
	b.prev = nil
	b.next = nil
	b.archived = false
	b.closed = false
	b.data = b.data[:0]
	return b
}

// write appends a timestamped value to the buffer chain.
//
// Returns the head buffer (which may be newly created if capacity was reached).
// Timestamps older than the buffer's start are rejected. If the calculated index
// exceeds capacity, a new buffer is allocated and linked as the new head.
//
// Missing timestamps are automatically filled with NaN values to maintain alignment.
// Overwrites are allowed if the index is already within the existing data slice.
//
// Parameters:
//   - ts:    Unix timestamp in seconds
//   - value: Metric value (can be schema.NaN for missing data)
//
// Returns:
//   - *buffer: The new head buffer (same as b if no new buffer created)
//   - error:   Non-nil if timestamp is before buffer start
func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
	if ts < b.start {
		return nil, errors.New("[METRICSTORE]> cannot write value to buffer from past")
	}

	// idx := int((ts - b.start + (b.frequency / 3)) / b.frequency)
	idx := int((ts - b.start) / b.frequency)
	if idx >= cap(b.data) {
		newbuf := newBuffer(ts, b.frequency)
		newbuf.prev = b
		b.next = newbuf
		b = newbuf
		idx = 0
	}

	// Overwriting value or writing value from past
	if idx < len(b.data) {
		b.data[idx] = value
		return b, nil
	}

	// Fill up unwritten slots with NaN
	for i := len(b.data); i < idx; i++ {
		b.data = append(b.data, schema.NaN)
	}

	b.data = append(b.data, value)
	return b, nil
}

func (b *buffer) end() int64 {
	return b.firstWrite() + int64(len(b.data))*b.frequency
}

func (b *buffer) firstWrite() int64 {
	return b.start + (b.frequency / 2)
}

// read retrieves time-series data from the buffer chain for the specified time range.
//
// Traverses the buffer chain backwards (via prev links) if 'from' precedes the current
// buffer's start. Missing data points are represented as NaN. Values are accumulated
// into the provided 'data' slice (using +=, so caller must zero-initialize if needed).
//
// The function adjusts the actual time range returned if data is unavailable at the
// boundaries (returned via adjusted from/to timestamps).
//
// Parameters:
//   - from: Start timestamp (Unix seconds)
//   - to:   End timestamp (Unix seconds, exclusive)
//   - data: Pre-allocated slice to accumulate results (must be large enough)
//
// Returns:
//   - []schema.Float: Slice of data (may be shorter than input 'data' slice)
//   - int64:          Actual start timestamp with available data
//   - int64:          Actual end timestamp (exclusive)
//   - error:          Non-nil on failure
//
// Panics if 'data' slice is too small to hold all values in [from, to).
func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) {
	// Walk back to the buffer that covers 'from', adjusting if we hit the oldest.
	for from < b.firstWrite() {
		if b.prev == nil {
			from = b.firstWrite()
			break
		}
		b = b.prev
	}

	i := 0
	t := from


Let me check the git history of the WAL implementation and how WAL messages are produced, to pinpoint the intended rotation design.


git log --oneline -15 -- pkg/metricstore/walCheckpoint.go pkg/metricstore/checkpoint.go; echo "=== lineprotocol WAL send ==="; grep -n "SendWALMessage\|WALMessage\|walDropped\|FileFormat" pkg/metricstore/lineprotocol.go


641dc0e3 Run gofumpt
66707bbf Update metricstore documentation
fc47b12f fix: Pause WAL writes during binary checkpoint to prevent message drops
937984d1 fix: WAL rotation skipped for all nodes due to non-blocking send on small channel
cc3d03bb fix: Unbound growth of wal files in case of checkpointing error
ac0a4cc3 Increase shutdown timeouts and WAL flush interval
97d65a9e Fix bugs in WAL journal pipeline
e7598100 Add shutdown timings. Do not drain WAL buffers on shutdown
6f7dda53 Cleanup
0325d9e8 fix: Increase throughput for WAL writers
bf1a8a17 fix: Shard WAL consumer for higher throughput
50aed595 fix: metricstore NATS contention
39ab1278 Make checkpointInterval an option config option again.
b214e175 Add buffered I/O to WAL writes and fix MemoryCap comment
39635ea1 Cleanup metricstore options
=== lineprotocol WAL send ===
18://     goroutine via the WALMessages channel for durable write-ahead logging.
128:// walDropped counts WAL messages dropped due to a full channel.
130:var walDropped atomic.Int64
141:	// selector is the sub-device path passed to WriteToLevel and WALMessage
198:// sent to WALMessages so the WAL staging goroutine can persist it durably
377:		if Keys.Checkpoints.FileFormat == "wal" {
378:			msg := &WALMessage{
386:			if !SendWALMessage(msg) {
389:				if dropped := walDropped.Add(1); dropped%10000 == 1 {


git show cc3d03bb --stat; echo "=========="; git show cc3d03bb -- pkg/metricstore/checkpoint.go | head -120


commit cc3d03bb5b78c26b3dd2e04279b6206ed3753034
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Sat Mar 28 06:26:21 2026 +0100

    fix: Unbound growth of wal files in case of checkpointing error

    Entire-Checkpoint: 95a89a7127c5

 ReleaseNotes.md                  | 10 ++++++++++
 pkg/metricstore/checkpoint.go    |  3 ++-
 pkg/metricstore/config.go        |  7 +++++--
 pkg/metricstore/configSchema.go  |  5 +++++
 pkg/metricstore/walCheckpoint.go | 40 ++++++++++++++++++++++++++++++++++------
 5 files changed, 56 insertions(+), 9 deletions(-)
==========
commit cc3d03bb5b78c26b3dd2e04279b6206ed3753034
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Sat Mar 28 06:26:21 2026 +0100

    fix: Unbound growth of wal files in case of checkpointing error

    Entire-Checkpoint: 95a89a7127c5

diff --git a/pkg/metricstore/checkpoint.go b/pkg/metricstore/checkpoint.go
index 3627a67b..2dcb5733 100644
--- a/pkg/metricstore/checkpoint.go
+++ b/pkg/metricstore/checkpoint.go
@@ -129,7 +129,8 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
 					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
 					if err != nil {
 						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
-					} else {
+					}
+					if n > 0 {
 						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
 						lastCheckpointMu.Lock()
 						lastCheckpoint = now


I found the smoking gun. Let me verify that `RotateWALFilesAfterShutdown` is never actually called anywhere.


grep -rn "RotateWALFilesAfterShutdown\|RotateWALFiles\b" pkg/ internal/ cmd/ 2>/dev/null; echo "=== Shutdown callers ==="; grep -rn "metricstore.Shutdown\|metricStore.Shutdown\|\.Shutdown()" cmd/ internal/ pkg/metricstore/*.go 2>/dev/null | grep -i metric


pkg/metricstore/checkpoint.go:143:						RotateWALFiles(hostDirs)
pkg/metricstore/walCheckpoint.go:359:// RotateWALFiles sends rotation requests for the given host directories
pkg/metricstore/walCheckpoint.go:365:func RotateWALFiles(hostDirs []string) {
pkg/metricstore/walCheckpoint.go:403:// RotateWALFilesAfterShutdown directly removes current.wal files for the given
pkg/metricstore/walCheckpoint.go:405:// exited and the channel-based RotateWALFiles is no longer safe to call.
pkg/metricstore/walCheckpoint.go:406:func RotateWALFilesAfterShutdown(hostDirs []string) {
=== Shutdown callers ===
cmd/cc-backend/server.go:484:				metricstore.Shutdown()


I now have a definitive root cause. Let me write the plan.

The mechanism is clear:
- **Periodic checkpoint** (`Checkpointing`, default every 12h): writes `.bin` snapshots, then calls `RotateWALFiles(successDirs)` → WAL reset works here.
- **Force-rotate** when `ws.size >= MaxWALSize` — but `MaxWALSize` defaults to `0` (unlimited), so never fires by default.
- **Shutdown** (`Shutdown()`): writes a final `.bin` snapshot via `ToCheckpointWAL`, but **discards the `successDirs` return (`_`) and never rotates the WAL**. The comment "WAL files are deleted per-host inside ToCheckpointWAL workers" is false — `ToCheckpointWAL`/`toCheckpointBinary` only write snapshots. The purpose-built helper `RotateWALFilesAfterShutdown` exists but is **never called anywhere**.

So after every clean shutdown the `current.wal` files survive, the next startup replays them *and keeps appending to the same files*, and they are never reset until a (12h-default) periodic checkpoint happens to run. That is exactly "not reset and continuously grow after a start."


Write

# Fix: WAL files are never reset on shutdown (continuous growth across restarts)

## Context

The metric store (`pkg/metricstore`) uses a WAL + periodic binary-snapshot
checkpoint scheme for crash recovery. Each host has an append-only
`checkpoints/<cluster>/<host>/current.wal` that records every metric write, plus
periodic `<ts>.bin` full-tree snapshots. On startup the newest `.bin` is loaded
and `current.wal` is replayed on top.

A `current.wal` is only meant to hold writes *since the last snapshot*. Once a
snapshot is written, the WAL is redundant and must be rotated (deleted). The user
reports that WAL files are never reset and grow without bound after a start.

## Root cause

WAL rotation happens in exactly these places today:

1. **Periodic checkpoint** — `Checkpointing()` in `pkg/metricstore/checkpoint.go:128-145`
   writes snapshots via `ToCheckpointWAL`, then calls `RotateWALFiles(hostDirs)`
   for the hosts that got a fresh snapshot. This path works, but the default
   `checkpoint-interval` is **12h**, so the WAL grows for up to 12h between resets.
2. **Force-rotate on size** — in `WALStaging`/`processMsg`
   (`pkg/metricstore/walCheckpoint.go:255`) when `ws.size >= MaxWALSize`. But
   `MaxWALSize` (`config.go:69`) defaults to `0` = unlimited, so this never fires
   unless explicitly configured.
3. **Shutdown** — `Shutdown()` in `pkg/metricstore/metricstore.go:310-315`.

The **shutdown path is broken**. It writes a final snapshot but never rotates the
WAL:

```go
if Keys.Checkpoints.FileFormat == "wal" {
    // WAL files are deleted per-host inside ToCheckpointWAL workers.   <-- FALSE
    files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
}
```

- The comment is incorrect: `ToCheckpointWAL` / `toCheckpointBinary`
  (`walCheckpoint.go:667`, `:768`) only write `.bin` snapshots; they do **not**
  touch `current.wal`.
- The second return value (`successDirs` — hosts that got a fresh snapshot) is
  discarded with `_`.
- The purpose-built helper `RotateWALFilesAfterShutdown(hostDirs)`
  (`walCheckpoint.go:406`), which directly `os.Remove`s `current.wal` files after
  the staging goroutines have exited, is **defined but never called** (verified by
  grep across `cmd/`, `internal/`, `pkg/`).

Consequence: after each clean shutdown the `current.wal` files remain on disk. On
the next startup they are replayed (redundantly, since the `.bin` already has that
data) and then **appended to again** (the staging goroutine opens the existing
file with `O_APPEND`). Across restarts the WAL grows monotonically and is only
ever reset if a 12h periodic checkpoint happens to run. This is the reported
"not reset / continuously grows after a start" behavior.

Note on ordering: in `Shutdown()` the WAL staging goroutines are already stopped
(channels closed + `WaitForWALStagingDrain()`) *before* the final
`ToCheckpointWAL`. So the channel-based `RotateWALFiles` is unsafe here — that is
exactly why the direct-removal `RotateWALFilesAfterShutdown` variant exists.

## Fix

In `pkg/metricstore/metricstore.go`, `Shutdown()` (the `FileFormat == "wal"`
branch, ~line 310): capture the `successDirs` return from `ToCheckpointWAL` and
rotate those WAL files after the snapshot is durably written.

```go
if Keys.Checkpoints.FileFormat == "wal" {
    var successDirs []string
    files, successDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
    // Staging goroutines have already exited, so remove the now-redundant
    // current.wal files directly (channel-based RotateWALFiles is unsafe here).
    RotateWALFilesAfterShutdown(successDirs)
} else {
    files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
}
```

Rotate **only `successDirs`** (hosts that received a fresh snapshot), mirroring the
periodic path. A host that produced no snapshot (`ErrNoNewArchiveData`) had no new
unarchived data for the window, so its WAL is left untouched to avoid dropping any
record not yet captured by a snapshot. Remove/replace the misleading comment.

### Files
- `pkg/metricstore/metricstore.go` — `Shutdown()` WAL branch (only change needed).

### Reused existing code
- `RotateWALFilesAfterShutdown([]string)` — `pkg/metricstore/walCheckpoint.go:406`
  (already implemented; just needs to be wired in).
- `successDirs` return of `ToCheckpointWAL` — `walCheckpoint.go:667`.

## Verification

1. Build: `go build ./...`
2. Targeted unit test (add to `pkg/metricstore`): write a few metrics so a host
   WAL exists, run a checkpoint + `Shutdown()` equivalent, then assert
   `current.wal` no longer exists (or is recreated empty) for hosts that were
   snapshotted. Run `go test ./pkg/metricstore/...`.
3. Manual end-to-end:
   - Configure `checkpoints.file-format: "wal"` (default) with a short
     `checkpoint-interval` is *not* required for this test.
   - Start `cc-backend`, ingest some metrics, confirm
     `var/checkpoints/<cluster>/<host>/current.wal` grows.
   - Stop the server (graceful shutdown) and confirm a new `<ts>.bin` appears and
     `current.wal` is gone for active hosts.
   - Restart, ingest more, stop again — confirm the WAL does not accumulate the
     previous run's bytes (size resets each clean shutdown rather than growing
     monotonically).

## Out of scope / follow-ups (mention only)
- The 12h default `checkpoint-interval` still bounds in-run WAL growth; operators
  who want tighter bounds can lower it or set `checkpoints.max-wal-size`. No code
  change proposed here unless requested.


File created successfully at: /Users/jan/.claude/plans/this-application-uses-a-shiny-flame.md (file state is current in your context — no need to Read it back)

# Fix: WAL files are never reset on shutdown (continuous growth across restarts)

## Context

The metric store (`pkg/metricstore`) uses a WAL + periodic binary-snapshot
checkpoint scheme for crash recovery. Each host has an append-only
`checkpoints/<cluster>/<host>/current.wal` that records every metric write, plus
periodic `<ts>.bin` full-tree snapshots. On startup the newest `.bin` is loaded
and `current.wal` is replayed on top.

A `current.wal` is only meant to hold writes *since the last snapshot*. Once a
snapshot is written, the WAL is redundant and must be rotated (deleted). The user
reports that WAL files are never reset and grow without bound after a start.

## Root cause

WAL rotation happens in exactly these places today:

1. **Periodic checkpoint** — `Checkpointing()` in `pkg/metricstore/checkpoint.go:128-145`
   writes snapshots via `ToCheckpointWAL`, then calls `RotateWALFiles(hostDirs)`
   for the hosts that got a fresh snapshot. This path works, but the default
   `checkpoint-interval` is **12h**, so the WAL grows for up to 12h between resets.
2. **Force-rotate on size** — in `WALStaging`/`processMsg`
   (`pkg/metricstore/walCheckpoint.go:255`) when `ws.size >= MaxWALSize`. But
   `MaxWALSize` (`config.go:69`) defaults to `0` = unlimited, so this never fires
   unless explicitly configured.
3. **Shutdown** — `Shutdown()` in `pkg/metricstore/metricstore.go:310-315`.

The **shutdown path is broken**. It writes a final snapshot but never rotates the
WAL:

```go
if Keys.Checkpoints.FileFormat == "wal" {
    // WAL files are deleted per-host inside ToCheckpointWAL workers.   <-- FALSE
    files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
}
```

- The comment is incorrect: `ToCheckpointWAL` / `toCheckpointBinary`
  (`walCheckpoint.go:667`, `:768`) only write `.bin` snapshots; they do **not**
  touch `current.wal`.
- The second return value (`successDirs` — hosts that got a fresh snapshot) is
  discarded with `_`.
- The purpose-built helper `RotateWALFilesAfterShutdown(hostDirs)`
  (`walCheckpoint.go:406`), which directly `os.Remove`s `current.wal` files after
  the staging goroutines have exited, is **defined but never called** (verified by
  grep across `cmd/`, `internal/`, `pkg/`).

Consequence: after each clean shutdown the `current.wal` files remain on disk. On
the next startup they are replayed (redundantly, since the `.bin` already has that
data) and then **appended to again** (the staging goroutine opens the existing
file with `O_APPEND`). Across restarts the WAL grows monotonically and is only
ever reset if a 12h periodic checkpoint happens to run. This is the reported
"not reset / continuously grows after a start" behavior.

Note on ordering: in `Shutdown()` the WAL staging goroutines are already stopped
(channels closed + `WaitForWALStagingDrain()`) *before* the final
`ToCheckpointWAL`. So the channel-based `RotateWALFiles` is unsafe here — that is
exactly why the direct-removal `RotateWALFilesAfterShutdown` variant exists.

## Fix

In `pkg/metricstore/metricstore.go`, `Shutdown()` (the `FileFormat == "wal"`
branch, ~line 310): capture the `successDirs` return from `ToCheckpointWAL` and
rotate those WAL files after the snapshot is durably written.

```go
if Keys.Checkpoints.FileFormat == "wal" {
    var successDirs []string
    files, successDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
    // Staging goroutines have already exited, so remove the now-redundant
    // current.wal files directly (channel-based RotateWALFiles is unsafe here).
    RotateWALFilesAfterShutdown(successDirs)
} else {
    files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
}
```

Rotate **only `successDirs`** (hosts that received a fresh snapshot), mirroring the
periodic path. A host that produced no snapshot (`ErrNoNewArchiveData`) had no new
unarchived data for the window, so its WAL is left untouched to avoid dropping any
record not yet captured by a snapshot. Remove/replace the misleading comment.

### Files
- `pkg/metricstore/metricstore.go` — `Shutdown()` WAL branch (only change needed).

### Reused existing code
- `RotateWALFilesAfterShutdown([]string)` — `pkg/metricstore/walCheckpoint.go:406`
  (already implemented; just needs to be wired in).
- `successDirs` return of `ToCheckpointWAL` — `walCheckpoint.go:667`.

## Verification

1. Build: `go build ./...`
2. Targeted unit test (add to `pkg/metricstore`): write a few metrics so a host
   WAL exists, run a checkpoint + `Shutdown()` equivalent, then assert
   `current.wal` no longer exists (or is recreated empty) for hosts that were
   snapshotted. Run `go test ./pkg/metricstore/...`.
3. Manual end-to-end:
   - Configure `checkpoints.file-format: "wal"` (default) with a short
     `checkpoint-interval` is *not* required for this test.
   - Start `cc-backend`, ingest some metrics, confirm
     `var/checkpoints/<cluster>/<host>/current.wal` grows.
   - Stop the server (graceful shutdown) and confirm a new `<ts>.bin` appears and
     `current.wal` is gone for active hosts.
   - Restart, ingest more, stop again — confirm the WAL does not accumulate the
     previous run's bytes (size resets each clean shutdown rather than growing
     monotonically).

## Out of scope / follow-ups (mention only)
- The 12h default `checkpoint-interval` still bounds in-run WAL growth; operators
  who want tighter bounds can lower it or set `checkpoints.max-wal-size`. No code
  change proposed here unless requested.


ToolSearch

Question: select:ExitPlanMode


Question: select:ExitPlanMode


## Analysis: when are WAL files reset

WAL rotation (deletion of `current.wal`) happens in only three places:

1. **Periodic checkpoint** — `Checkpointing()` (`checkpoint.go:128-145`) writes `.bin` snapshots, then `RotateWALFiles(successDirs)`. ✅ works, but default `checkpoint-interval` is **12h**.
2. **Size cap** — force-rotate when `ws.size >= MaxWALSize` (`walCheckpoint.go:255`). But `max-wal-size` defaults to **0 = unlimited**, so it never fires by default.
3. **Shutdown** — `Shutdown()` (`metricstore.go:310-315`). **Broken.**

The bug is in shutdown. It writes a final snapshot but never rotates:

```go
// WAL files are deleted per-host inside ToCheckpointWAL workers.   ← FALSE
files, _, err = ms.ToCheckpointWAL(...)
```

`ToCheckpointWAL` only writes `.bin` files — it does not touch `current.wal`. The `successDirs` return is thrown away (`_`), and the dedicated helper `RotateWALFilesAfterShutdown()` (`walCheckpoint.go:406`) exists but is **never called anywhere** (confirmed by grep). So after each clean shutdown the `current.wal` files survive; the next startup replays them *and keeps appending to the same files* (opened `O_APPEND`). They only ever reset if a 12h periodic checkpoint happens to run — exactly the "not reset / continuously grows after a start" you observed.

The fix is small: in `Shutdown()`, capture `successDirs` and call `RotateWALFilesAfterShutdown(successDirs)` after the final snapshot (the staging goroutines are already stopped, which is why the direct-removal variant exists). Full details, verification steps, and reasoning are in the plan.


ExitPlanMode

# Fix: WAL files are never reset on shutdown (continuous growth across restarts)

## Context

The metric store (`pkg/metricstore`) uses a WAL + periodic binary-snapshot
checkpoint scheme for crash recovery. Each host has an append-only
`checkpoints/<cluster>/<host>/current.wal` that records every metric write, plus
periodic `<ts>.bin` full-tree snapshots. On startup the newest `.bin` is loaded
and `current.wal` is replayed on top.

A `current.wal` is only meant to hold writes *since the last snapshot*. Once a
snapshot is written, the WAL is redundant and must be rotated (deleted). The user
reports that WAL files are never reset and grow without bound after a start.

## Root cause

WAL rotation happens in exactly these places today:

1. **Periodic checkpoint** — `Checkpointing()` in `pkg/metricstore/checkpoint.go:128-145`
   writes snapshots via `ToCheckpointWAL`, then calls `RotateWALFiles(hostDirs)`
   for the hosts that got a fresh snapshot. This path works, but the default
   `checkpoint-interval` is **12h**, so the WAL grows for up to 12h between resets.
2. **Force-rotate on size** — in `WALStaging`/`processMsg`
   (`pkg/metricstore/walCheckpoint.go:255`) when `ws.size >= MaxWALSize`. But
   `MaxWALSize` (`config.go:69`) defaults to `0` = unlimited, so this never fires
   unless explicitly configured.
3. **Shutdown** — `Shutdown()` in `pkg/metricstore/metricstore.go:310-315`.

The **shutdown path is broken**. It writes a final snapshot but never rotates the
WAL:

```go
if Keys.Checkpoints.FileFormat == "wal" {
    // WAL files are deleted per-host inside ToCheckpointWAL workers.   <-- FALSE
    files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
}
```

- The comment is incorrect: `ToCheckpointWAL` / `toCheckpointBinary`
  (`walCheckpoint.go:667`, `:768`) only write `.bin` snapshots; they do **not**
  touch `current.wal`.
- The second return value (`successDirs` — hosts that got a fresh snapshot) is
  discarded with `_`.
- The purpose-built helper `RotateWALFilesAfterShutdown(hostDirs)`
  (`walCheckpoint.go:406`), which directly `os.Remove`s `current.wal` files after
  the staging goroutines have exited, is **defined but never called** (verified by
  grep across `cmd/`, `internal/`, `pkg/`).

Consequence: after each clean shutdown the `current.wal` files remain on disk. On
the next startup they are replayed (redundantly, since the `.bin` already has that
data) and then **appended to again** (the staging goroutine opens the existing
file with `O_APPEND`). Across restarts the WAL grows monotonically and is only
ever reset if a 12h periodic checkpoint happens to run. This is the reported
"not reset / continuously grows after a start" behavior.

Note on ordering: in `Shutdown()` the WAL staging goroutines are already stopped
(channels closed + `WaitForWALStagingDrain()`) *before* the final
`ToCheckpointWAL`. So the channel-based `RotateWALFiles` is unsafe here — that is
exactly why the direct-removal `RotateWALFilesAfterShutdown` variant exists.

## Fix

In `pkg/metricstore/metricstore.go`, `Shutdown()` (the `FileFormat == "wal"`
branch, ~line 310): capture the `successDirs` return from `ToCheckpointWAL` and
rotate those WAL files after the snapshot is durably written.

```go
if Keys.Checkpoints.FileFormat == "wal" {
    var successDirs []string
    files, successDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
    // Staging goroutines have already exited, so remove the now-redundant
    // current.wal files directly (channel-based RotateWALFiles is unsafe here).
    RotateWALFilesAfterShutdown(successDirs)
} else {
    files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
}
```

Rotate **only `successDirs`** (hosts that received a fresh snapshot), mirroring the
periodic path. A host that produced no snapshot (`ErrNoNewArchiveData`) had no new
unarchived data for the window, so its WAL is left untouched to avoid dropping any
record not yet captured by a snapshot. Remove/replace the misleading comment.

### Files
- `pkg/metricstore/metricstore.go` — `Shutdown()` WAL branch (only change needed).

### Reused existing code
- `RotateWALFilesAfterShutdown([]string)` — `pkg/metricstore/walCheckpoint.go:406`
  (already implemented; just needs to be wired in).
- `successDirs` return of `ToCheckpointWAL` — `walCheckpoint.go:667`.

## Verification

1. Build: `go build ./...`
2. Targeted unit test (add to `pkg/metricstore`): write a few metrics so a host
   WAL exists, run a checkpoint + `Shutdown()` equivalent, then assert
   `current.wal` no longer exists (or is recreated empty) for hosts that were
   snapshotted. Run `go test ./pkg/metricstore/...`.
3. Manual end-to-end:
   - Configure `checkpoints.file-format: "wal"` (default) with a short
     `checkpoint-interval` is *not* required for this test.
   - Start `cc-backend`, ingest some metrics, confirm
     `var/checkpoints/<cluster>/<host>/current.wal` grows.
   - Stop the server (graceful shutdown) and confirm a new `<ts>.bin` appears and
     `current.wal` is gone for active hosts.
   - Restart, ingest more, stop again — confirm the WAL does not accumulate the
     previous run's bytes (size resets each clean shutdown rather than growing
     monotonically).

## Out of scope / follow-ups (mention only)
- The 12h default `checkpoint-interval` still bounds in-run WAL growth; operators
  who want tighter bounds can lower it or set `checkpoints.max-wal-size`. No code
  change proposed here unless requested.


User has approved your plan. You can now start coding. Start with updating your todo list if applicable

Your plan has been saved to: /Users/jan/.claude/plans/this-application-uses-a-shiny-flame.md
You can refer back to it if needed during implementation.

## Approved Plan:
# Fix: WAL files are never reset on shutdown (continuous growth across restarts)

## Context

The metric store (`pkg/metricstore`) uses a WAL + periodic binary-snapshot
checkpoint scheme for crash recovery. Each host has an append-only
`checkpoints/<cluster>/<host>/current.wal` that records every metric write, plus
periodic `<ts>.bin` full-tree snapshots. On startup the newest `.bin` is loaded
and `current.wal` is replayed on top.

A `current.wal` is only meant to hold writes *since the last snapshot*. Once a
snapshot is written, the WAL is redundant and must be rotated (deleted). The user
reports that WAL files are never reset and grow without bound after a start.

## Root cause

WAL rotation happens in exactly these places today:

1. **Periodic checkpoint** — `Checkpointing()` in `pkg/metricstore/checkpoint.go:128-145`
   writes snapshots via `ToCheckpointWAL`, then calls `RotateWALFiles(hostDirs)`
   for the hosts that got a fresh snapshot. This path works, but the default
   `checkpoint-interval` is **12h**, so the WAL grows for up to 12h between resets.
2. **Force-rotate on size** — in `WALStaging`/`processMsg`
   (`pkg/metricstore/walCheckpoint.go:255`) when `ws.size >= MaxWALSize`. But
   `MaxWALSize` (`config.go:69`) defaults to `0` = unlimited, so this never fires
   unless explicitly configured.
3. **Shutdown** — `Shutdown()` in `pkg/metricstore/metricstore.go:310-315`.

The **shutdown path is broken**. It writes a final snapshot but never rotates the
WAL:

```go
if Keys.Checkpoints.FileFormat == "wal" {
    // WAL files are deleted per-host inside ToCheckpointWAL workers.   <-- FALSE
    files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
}
```

- The comment is incorrect: `ToCheckpointWAL` / `toCheckpointBinary`
  (`walCheckpoint.go:667`, `:768`) only write `.bin` snapshots; they do **not**
  touch `current.wal`.
- The second return value (`successDirs` — hosts that got a fresh snapshot) is
  discarded with `_`.
- The purpose-built helper `RotateWALFilesAfterShutdown(hostDirs)`
  (`walCheckpoint.go:406`), which directly `os.Remove`s `current.wal` files after
  the staging goroutines have exited, is **defined but never called** (verified by
  grep across `cmd/`, `internal/`, `pkg/`).

Consequence: after each clean shutdown the `current.wal` files remain on disk. On
the next startup they are replayed (redundantly, since the `.bin` already has that
data) and then **appended to again** (the staging goroutine opens the existing
file with `O_APPEND`). Across restarts the WAL grows monotonically and is only
ever reset if a 12h periodic checkpoint happens to run. This is the reported
"not reset / continuously grows after a start" behavior.

Note on ordering: in `Shutdown()` the WAL staging goroutines are already stopped
(channels closed + `WaitForWALStagingDrain()`) *before* the final
`ToCheckpointWAL`. So the channel-based `RotateWALFiles` is unsafe here — that is
exactly why the direct-removal `RotateWALFilesAfterShutdown` variant exists.

## Fix

In `pkg/metricstore/metricstore.go`, `Shutdown()` (the `FileFormat == "wal"`
branch, ~line 310): capture the `successDirs` return from `ToCheckpointWAL` and
rotate those WAL files after the snapshot is durably written.

```go
if Keys.Checkpoints.FileFormat == "wal" {
    var successDirs []string
    files, successDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
    // Staging goroutines have already exited, so remove the now-redundant
    // current.wal files directly (channel-based RotateWALFiles is unsafe here).
    RotateWALFilesAfterShutdown(successDirs)
} else {
    files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
}
```

Rotate **only `successDirs`** (hosts that received a fresh snapshot), mirroring the
periodic path. A host that produced no snapshot (`ErrNoNewArchiveData`) had no new
unarchived data for the window, so its WAL is left untouched to avoid dropping any
record not yet captured by a snapshot. Remove/replace the misleading comment.

### Files
- `pkg/metricstore/metricstore.go` — `Shutdown()` WAL branch (only change needed).

### Reused existing code
- `RotateWALFilesAfterShutdown([]string)` — `pkg/metricstore/walCheckpoint.go:406`
  (already implemented; just needs to be wired in).
- `successDirs` return of `ToCheckpointWAL` — `walCheckpoint.go:667`.

## Verification

1. Build: `go build ./...`
2. Targeted unit test (add to `pkg/metricstore`): write a few metrics so a host
   WAL exists, run a checkpoint + `Shutdown()` equivalent, then assert
   `current.wal` no longer exists (or is recreated empty) for hosts that were
   snapshotted. Run `go test ./pkg/metricstore/...`.
3. Manual end-to-end:
   - Configure `checkpoints.file-format: "wal"` (default) with a short
     `checkpoint-interval` is *not* required for this test.
   - Start `cc-backend`, ingest some metrics, confirm
     `var/checkpoints/<cluster>/<host>/current.wal` grows.
   - Stop the server (graceful shutdown) and confirm a new `<ts>.bin` appears and
     `current.wal` is gone for active hosts.
   - Restart, ingest more, stop again — confirm the WAL does not accumulate the
     previous run's bytes (size resets each clean shutdown rather than growing
     monotonically).

## Out of scope / follow-ups (mention only)
- The 12h default `checkpoint-interval` still bounds in-run WAL growth; operators
  who want tighter bounds can lower it or set `checkpoints.max-wal-size`. No code
  change proposed here unless requested.


Auto mode lets Claude handle permission prompts automatically — Claude checks each tool call for risky actions and prompt injection before executing. Actions Claude identifies as safe are executed, while actions Claude identifies as risky are blocked and Claude may try a different approach. Ideal for long-running tasks. Sessions are slightly more expensive. Claude can make mistakes that allow harmful commands to run, it's recommended to only use in isolated environments. Shift+Tab to change mode.


Let me read the exact lines in `Shutdown()` and apply the fix.


299		cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
300		checkpointStart := time.Now()
301		var files int
302		var err error
303
304		ms := GetMemoryStore()
305
306		lastCheckpointMu.Lock()
307		from := lastCheckpoint
308		lastCheckpointMu.Unlock()
309
310		if Keys.Checkpoints.FileFormat == "wal" {
311			// WAL files are deleted per-host inside ToCheckpointWAL workers.
312			files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
313		} else {
314			files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
315		}
316
317		if err != nil {
318			cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
319		}
320		cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))

	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
	checkpointStart := time.Now()
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		// WAL files are deleted per-host inside ToCheckpointWAL workers.
		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))


Edit

	if Keys.Checkpoints.FileFormat == "wal" {
		// WAL files are deleted per-host inside ToCheckpointWAL workers.
		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	} else {

	if Keys.Checkpoints.FileFormat == "wal" {
		var successDirs []string
		files, successDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
		// The final binary snapshot now captures all in-memory data for these
		// hosts, making their current.wal redundant. The staging goroutines have
		// already exited, so remove the WAL files directly (the channel-based
		// RotateWALFiles is no longer safe to call). Without this, current.wal
		// files survive shutdown and keep growing across restarts.
		RotateWALFilesAfterShutdown(successDirs)
	} else {