func Shutdown() {
	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}

	if Keys.Checkpoints.FileFormat == "wal" {
		for _, ch := range walShardChs {
			close(ch)
		}
	}

	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		var hostDirs []string
		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
		if err == nil {
			RotateWALFilesAfterShutdown(hostDirs)
		}
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
}

func Shutdown() {
	totalStart := time.Now()

	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}
	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))

	if Keys.Checkpoints.FileFormat == "wal" {
		for _, ch := range walShardChs {
			close(ch)
		}
		drainStart := time.Now()
		WaitForWALStagingDrain()
		cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
	}

	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
	checkpointStart := time.Now()
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		// WAL files are deleted per-host inside ToCheckpointWAL workers.
		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go has been updated successfully.

func Shutdown() {
	totalStart := time.Now()

	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}
	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))

	if Keys.Checkpoints.FileFormat == "wal" {
		for _, ch := range walShardChs {
			close(ch)
		}
		drainStart := time.Now()
		WaitForWALStagingDrain()
		cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
	}

	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
	checkpointStart := time.Now()
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		// WAL files are deleted per-host inside ToCheckpointWAL workers.
		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides an efficient in-memory time-series metric storage system
// with support for hierarchical data organization, checkpointing, and archiving.
//
// The package organizes metrics in a tree structure (cluster → host → component) and
// provides concurrent read/write access to metric data with configurable aggregation strategies.
// Background goroutines handle periodic checkpointing (JSON or Avro format), archiving old data,
// and enforcing retention policies.
//
// Key features:
//   - In-memory metric storage with configurable retention
//   - Hierarchical data organization (selectors)
//   - Concurrent checkpoint/archive workers
//   - Support for sum and average aggregation
//   - NATS integration for metric ingestion
package metricstore

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"runtime"
	"runtime/debug"
	"slices"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/config"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/resampler"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-lib/v2/util"
)

// GlobalState holds the global state for the metric store with thread-safe access.
type GlobalState struct {
	mu                sync.RWMutex
	lastRetentionTime int64
	selectorsExcluded bool
}

var (
	singleton  sync.Once
	msInstance *MemoryStore
	// shutdownFunc stores the context cancellation function created in Init
	// and is called during Shutdown to cancel all background goroutines
	shutdownFunc   context.CancelFunc
	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
	// Create a global instance
	state = &GlobalState{}
)

// NodeProvider provides information about nodes currently in use by running jobs.
//
// This interface allows metricstore to query job information without directly
// depending on the repository package, breaking the import cycle.
//
// Implementations should return nodes that are actively processing jobs started
// before the given timestamp. These nodes will be excluded from retention-based
// garbage collection to prevent data loss for jobs that are still running or
// recently completed.
type NodeProvider interface {
	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
	// that are currently in use by jobs that started before the given timestamp.
	//
	// Parameters:
	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
	//
	// Returns:
	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
	//   - Error if the query fails
	GetUsedNodes(ts int64) (map[string][]string, error)
}

// Metric represents a single metric data point to be written to the store.
type Metric struct {
	Name  string
	Value schema.Float
	// MetricConfig contains frequency and aggregation settings for this metric.
	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
	MetricConfig MetricConfig
}

// MemoryStore is the main in-memory time-series metric storage implementation.
//
// It organizes metrics in a hierarchical tree structure where each level represents
// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
// store multiple metrics as time-series buffers.
//
// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
// All public methods are safe for concurrent use.
type MemoryStore struct {
	Metrics      map[string]MetricConfig
	root         Level
	nodeProvider NodeProvider
}

// Init initializes the metric store from configuration and starts background workers.
//
// This function must be called exactly once before any other metricstore operations.
// It performs the following initialization steps:
//  1. Validates and decodes the metric store configuration
//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
//  3. Loads metric configurations from all registered clusters
//  4. Restores checkpoints within the retention window
//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
//  6. Optionally subscribes to NATS for real-time metric ingestion
//
// Parameters:
//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig)
//   - wg: WaitGroup that will be incremented for each background goroutine started
//
// The function will call cclog.Fatal on critical errors during initialization.
// Use Shutdown() to cleanly stop all background workers started by Init().
//
// Note: Signal handling must be implemented by the caller. Call Shutdown() when
// receiving termination signals to ensure checkpoint data is persisted.
func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
	startupTime := time.Now()

	if rawConfig != nil {
		config.Validate(configSchema, rawConfig)
		dec := json.NewDecoder(bytes.NewReader(rawConfig))
		dec.DisallowUnknownFields()
		if err := dec.Decode(&Keys); err != nil {
			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
		}
	}

	// Set NumWorkers from config or use default
	if Keys.NumWorkers <= 0 {
		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
	}
	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)

	// Pass the config.MetricStoreKeys
	InitMetrics(metrics)

	ms := GetMemoryStore()

	d, err := time.ParseDuration(Keys.RetentionInMemory)
	if err != nil {
		cclog.Fatal(err)
	}

	restoreFrom := startupTime.Add(-d)
	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
	if err != nil {
		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
	} else {
		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
	}

	// Try to use less memory by forcing a GC run here and then
	// lowering the target percentage. The default of 100 means
	// that only once the ratio of new allocations execeds the
	// previously active heap, a GC is triggered.
	// Forcing a GC here will set the "previously active heap"
	// to a minumum.
	// runtime.GC()

	ctx, shutdown := context.WithCancel(context.Background())

	Retention(wg, ctx)
	Checkpointing(wg, ctx)
	CleanUp(wg, ctx)
	WALStaging(wg, ctx)
	MemoryUsageTracker(wg, ctx)

	// Note: Signal handling has been removed from this function.
	// The caller is responsible for handling shutdown signals and calling
	// the shutdown() function when appropriate.
	// Store the shutdown function for later use by Shutdown()
	shutdownFuncMu.Lock()
	shutdownFunc = shutdown
	shutdownFuncMu.Unlock()

	if Keys.Subscriptions != nil {
		wg.Go(func() {
			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
				cclog.Fatal(err)
			}
		})
	}
}

// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
//
// This function must be called before GetMemoryStore() and can only be called once due to
// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
//
// Parameters:
//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
//
// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
//
// After this call, the global msInstance is ready for use via GetMemoryStore().
func InitMetrics(metrics map[string]MetricConfig) {
	singleton.Do(func() {
		offset := 0
		for key, cfg := range metrics {
			if cfg.Frequency == 0 {
				panic("[METRICSTORE]> invalid frequency")
			}

			metrics[key] = MetricConfig{
				Frequency:   cfg.Frequency,
				Aggregation: cfg.Aggregation,
				offset:      offset,
			}
			offset += 1
		}

		msInstance = &MemoryStore{
			root: Level{
				metrics:  make([]*buffer, len(metrics)),
				children: make(map[string]*Level),
			},
			Metrics: metrics,
		}
	})
}

// GetMemoryStore returns the singleton MemoryStore instance.
//
// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
//
// This function is safe for concurrent use after initialization.
func GetMemoryStore() *MemoryStore {
	if msInstance == nil {
		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
	}

	return msInstance
}

func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
	if metric, ok := ms.Metrics[metricName]; ok {
		return metric.Frequency, nil
	}
	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
}

// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
// This must be called during initialization to provide job state information
// for selective buffer retention during Free operations.
// If not set, the Free function will fall back to freeing all buffers.
func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
	ms.nodeProvider = provider
}

// Shutdown performs a graceful shutdown of the metric store.
//
// This function cancels all background goroutines started by Init() and writes
// a final checkpoint to disk before returning. It should be called when the
// application receives a termination signal.
//
// The function will:
//  1. Cancel the context to stop all background workers
//  2. Close the WAL messages channel if using WAL format
//  3. Write a final checkpoint to preserve in-memory data
//  4. Log any errors encountered during shutdown
//
// Note: This function blocks until the final checkpoint is written.
func Shutdown() {
	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}

	if Keys.Checkpoints.FileFormat == "wal" {
		for _, ch := range walShardChs {
			close(ch)
		}
	}

	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		var hostDirs []string
		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
		if err == nil {
			RotateWALFilesAfterShutdown(hostDirs)
		}
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
}

// Retention starts a background goroutine that periodically frees old metric data.
//
// This worker runs at half the retention interval and calls Free() to remove buffers
// older than the configured retention time. It respects the NodeProvider to preserve
// data for nodes with active jobs.
//
// Parameters:
//   - wg: WaitGroup to signal completion when context is cancelled
//   - ctx: Context for cancellation signal
//
// The goroutine exits when ctx is cancelled.
func Retention(wg *sync.WaitGroup, ctx context.Context) {
	ms := GetMemoryStore()

	wg.Go(func() {
		d, err := time.ParseDuration(Keys.RetentionInMemory)
		if err != nil {
			cclog.Fatal(err)
		}
		if d <= 0 {
			return
		}

		tickInterval := d / 2
		if tickInterval <= 0 {
			return
		}
		ticker := time.NewTicker(tickInterval)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				state.mu.Lock()

				t := time.Now().Add(-d)

				state.lastRetentionTime = t.Unix()

				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))

				freed, err := Free(ms, t)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
				} else {
					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
				}

				state.mu.Unlock()

				// Clean up the buffer pool
				bufferPool.Clean(state.lastRetentionTime)
			}
		}
	})
}

// MemoryUsageTracker starts a background goroutine that monitors memory usage.
//
// This worker checks actual process memory usage (via runtime.MemStats) periodically
// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
// performance issues.
//
// The tracker logs both actual memory usage (heap allocated) and metric data size for
// visibility into memory overhead from Go runtime structures and allocations.
//
// Parameters:
//   - wg: WaitGroup to signal completion when context is cancelled
//   - ctx: Context for cancellation signal
//
// The goroutine exits when ctx is cancelled.
func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
	ms := GetMemoryStore()

	wg.Go(func() {
		normalInterval := DefaultMemoryUsageTrackerInterval
		fastInterval := 30 * time.Second

		if normalInterval <= 0 {
			return
		}

		ticker := time.NewTicker(normalInterval)
		defer ticker.Stop()
		currentInterval := normalInterval

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				var mem runtime.MemStats
				runtime.ReadMemStats(&mem)
				actualMemoryGB := float64(mem.Alloc) / 1e9
				metricDataGB := ms.SizeInGB()
				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)

				freedExcluded := 0
				freedEmergency := 0
				var err error

				state.mu.RLock()
				lastRetention := state.lastRetentionTime
				selectorsExcluded := state.selectorsExcluded
				state.mu.RUnlock()

				if lastRetention != 0 && selectorsExcluded {
					freedExcluded, err = ms.Free(nil, lastRetention)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
					}

					if freedExcluded > 0 {
						debug.FreeOSMemory()
						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
					}
				}

				runtime.ReadMemStats(&mem)
				actualMemoryGB = float64(mem.Alloc) / 1e9

				if actualMemoryGB > float64(Keys.MemoryCap) {
					cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)

					// Use progressive time-based Free with increasing threshold
					// instead of ForceFree loop — fewer tree traversals, more effective
					d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
					if parseErr != nil {
						cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
					} else {
						thresholds := []float64{0.75, 0.5, 0.25}
						for _, fraction := range thresholds {
							threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
							freed, freeErr := ms.Free(nil, threshold.Unix())
							if freeErr != nil {
								cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
							}
							freedEmergency += freed

							bufferPool.Clear()
							runtime.GC()
							runtime.ReadMemStats(&mem)
							actualMemoryGB = float64(mem.Alloc) / 1e9

							if actualMemoryGB < float64(Keys.MemoryCap) {
								break
							}
						}
					}

					bufferPool.Clear()
					debug.FreeOSMemory()

					runtime.ReadMemStats(&mem)
					actualMemoryGB = float64(mem.Alloc) / 1e9

					if actualMemoryGB >= float64(Keys.MemoryCap) {
						cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
					} else {
						cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
					}
				}

				// Adaptive ticker: check more frequently when memory is high
				memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
				if memoryRatio > 0.8 && currentInterval != fastInterval {
					ticker.Reset(fastInterval)
					currentInterval = fastInterval
					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
				} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
					ticker.Reset(normalInterval)
					currentInterval = normalInterval
					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
				}
			}
		}
	})
}

// Free removes metric data older than the given time while preserving data for active nodes.
//
// This function implements intelligent retention by consulting the NodeProvider (if configured)
// to determine which nodes are currently in use by running jobs. Data for these nodes is
// preserved even if older than the retention time.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - t: Time threshold - buffers with data older than this will be freed
//
// Returns:
//   - Number of buffers freed
//   - Error if NodeProvider query fails
//
// Behavior:
//   - If no NodeProvider is set: frees all buffers older than t
//   - If NodeProvider returns empty map: frees all buffers older than t
//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
func Free(ms *MemoryStore, t time.Time) (int, error) {
	// If no NodeProvider is configured, free all buffers older than t
	if ms.nodeProvider == nil {
		return ms.Free(nil, t.Unix())
	}

	excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
	if err != nil {
		return 0, err
	}

	switch lenMap := len(excludeSelectors); lenMap {

	// If the length of the map returned by GetUsedNodes() is 0,
	// then use default Free method with nil selector
	case 0:
		state.selectorsExcluded = false
		return ms.Free(nil, t.Unix())

	// Else formulate selectors, exclude those from the map
	// and free the rest of the selectors
	default:
		state.selectorsExcluded = true
		selectors := GetSelectors(ms, excludeSelectors)
		return FreeSelected(ms, selectors, t)
	}
}

// FreeSelected frees buffers for specific selectors while preserving others.
//
// This function is used when we want to retain some specific nodes beyond the retention time.
// It iterates through the provided selectors and frees their associated buffers.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
//   - t: Time threshold for freeing buffers
//
// Returns the total number of buffers freed and any error encountered.
func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
	freed := 0

	for _, selector := range selectors {

		freedBuffers, err := ms.Free(selector, t.Unix())
		if err != nil {
			cclog.Errorf("error while freeing selected buffers: %#v", err)
		}
		freed += freedBuffers

	}

	return freed, nil
}

// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
//
// This function generates a list of selectors whose buffers should be freed by excluding
// selectors that correspond to nodes currently in use by running jobs.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
//
// Returns a list of selectors ([]string paths) that can be safely freed.
//
// Example:
//
//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
//	and excludeSelectors contains {"emmy": ["node001"]},
//	then only [["emmy", "node002"]] is returned.
func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
	allSelectors := ms.GetPaths(2)

	filteredSelectors := make([][]string, 0, len(allSelectors))

	for _, path := range allSelectors {
		if len(path) < 2 {
			continue
		}

		key := path[0]   // The "Key" (Level 1)
		value := path[1] // The "Value" (Level 2)

		exclude := false

		// Check if the key exists in our exclusion map
		if excludedValues, exists := excludeSelectors[key]; exists {
			// The key exists, now check if the specific value is in the exclusion list
			if slices.Contains(excludedValues, value) {
				exclude = true
			}
		}

		if !exclude {
			filteredSelectors = append(filteredSelectors, path)
		}
	}

	return filteredSelectors
}

// GetPaths returns a list of lists (paths) to the specified depth.
func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
	var results [][]string

	// Start recursion. Initial path is empty.
	// We treat Root as depth 0.
	ms.root.collectPaths(0, targetDepth, []string{}, &results)

	return results
}

// Write all values in `metrics` to the level specified by `selector` for time `ts`.
// Look at `findLevelOrCreate` for how selectors work.
func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
	var ok bool
	for i, metric := range metrics {
		if metric.MetricConfig.Frequency == 0 {
			metric.MetricConfig, ok = m.Metrics[metric.Name]
			if !ok {
				cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
				metric.MetricConfig.Frequency = 0
			}
			metrics[i] = metric
		}
	}

	return m.WriteToLevel(&m.root, selector, ts, metrics)
}

func (m *MemoryStore) GetLevel(selector []string) *Level {
	return m.root.findLevelOrCreate(selector, len(m.Metrics))
}

// WriteToLevel assumes that `minfo` in `metrics` is filled in
func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
	l = l.findLevelOrCreate(selector, len(m.Metrics))
	l.lock.Lock()
	defer l.lock.Unlock()

	for _, metric := range metrics {
		if metric.MetricConfig.Frequency == 0 {
			continue
		}

		b := l.metrics[metric.MetricConfig.offset]
		if b == nil {
			// First write to this metric and level
			b = newBuffer(ts, metric.MetricConfig.Frequency)
			l.metrics[metric.MetricConfig.offset] = b
		}

		nb, err := b.write(ts, metric.Value)
		if err != nil {
			return err
		}

		// Last write created a new buffer...
		if b != nb {
			l.metrics[metric.MetricConfig.offset] = nb
		}
	}
	return nil
}

// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
// The second and third return value are the actual from/to for the data. Those can be different from
// the range asked for if no data was available.
func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
	if from > to {
		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
	}

	minfo, ok := m.Metrics[metric]
	if !ok {
		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
	}

	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)

	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
		cdata, cfrom, cto, err := b.read(from, to, data)
		if err != nil {
			return err
		}

		if n == 0 {
			from, to = cfrom, cto
		} else if from != cfrom || to != cto || len(data) != len(cdata) {
			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
			if missingfront != 0 {
				return ErrDataDoesNotAlign
			}

			newlen := len(cdata) - missingback
			if newlen < 1 {
				return ErrDataDoesNotAlign
			}
			cdata = cdata[0:newlen]
			if len(cdata) != len(data) {
				return ErrDataDoesNotAlign
			}

			from, to = cfrom, cto
		}

		data = cdata
		n += 1
		return nil
	})

	if err != nil {
		return nil, 0, 0, 0, err
	} else if n == 0 {
		return nil, 0, 0, 0, ErrNoHostOrMetric
	} else if n > 1 {
		if minfo.Aggregation == AvgAggregation {
			normalize := 1. / schema.Float(n)
			for i := 0; i < len(data); i++ {
				data[i] *= normalize
			}
		} else if minfo.Aggregation != SumAggregation {
			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
		}
	}

	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
	if err != nil {
		return nil, 0, 0, 0, err
	}

	return data, from, to, resolution, nil
}

// Free releases all buffers for the selected level and all its children that
// contain only values older than `t`.
func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
	return m.GetLevel(selector).free(t)
}

// ForceFree unconditionally removes the oldest buffer from each metric chain.
func (m *MemoryStore) ForceFree() (int, error) {
	return m.GetLevel(nil).forceFree()
}

func (m *MemoryStore) FreeAll() error {
	m.root.lock.Lock()
	defer m.root.lock.Unlock()

	for k := range m.root.children {
		delete(m.root.children, k)
	}

	return nil
}

func (m *MemoryStore) SizeInBytes() int64 {
	return m.root.sizeInBytes()
}

func (m *MemoryStore) SizeInGB() float64 {
	return float64(m.root.sizeInBytes()) / 1e9
}

// ListChildren , given a selector, returns a list of all children of the level
// selected.
func (m *MemoryStore) ListChildren(selector []string) []string {
	lvl := &m.root
	for lvl != nil && len(selector) != 0 {
		lvl.lock.RLock()
		next := lvl.children[selector[0]]
		lvl.lock.RUnlock()
		lvl = next
		selector = selector[1:]
	}

	if lvl == nil {
		return nil
	}

	lvl.lock.RLock()
	defer lvl.lock.RUnlock()

	children := make([]string, 0, len(lvl.children))
	for child := range lvl.children {
		children = append(children, child)
	}

	return children
}

+  cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))

+    cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))

-  cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)

+  cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)

-    files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())

-      RotateWALFilesAfterShutdown(hostDirs)

+    // WAL files are deleted per-host inside ToCheckpointWAL workers.

+    files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())

-    cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())

+    cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())

-  cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)

+  cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))


Updated task #3 status


Updated task #4 status


Now update `server.go` `Shutdown()`:


Edit

func (s *Server) Shutdown(ctx context.Context) {
	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Run metricstore and archiver shutdown concurrently.
	// They are independent: metricstore writes .bin snapshots,
	// archiver flushes pending job archives.
	done := make(chan struct{})
	go func() {
		defer close(done)
		var wg sync.WaitGroup

		if ms := metricstore.GetMemoryStore(); ms != nil {
			wg.Go(func() {
				metricstore.Shutdown()
			})
		}

		wg.Go(func() {
			if err := archiver.Shutdown(10 * time.Second); err != nil {
				cclog.Warnf("Archiver shutdown: %v", err)
			}
		})

		wg.Wait()
	}()

	select {
	case <-done:
	case <-time.After(60 * time.Second):
		cclog.Warn("Shutdown deadline exceeded, forcing exit")
	}
}

func (s *Server) Shutdown(ctx context.Context) {
	shutdownStart := time.Now()

	natsStart := time.Now()
	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}
	cclog.Infof("Shutdown: NATS closed (%v)", time.Since(natsStart))

	httpStart := time.Now()
	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()
	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}
	cclog.Infof("Shutdown: HTTP server stopped (%v)", time.Since(httpStart))

	// Run metricstore and archiver shutdown concurrently.
	// They are independent: metricstore writes .bin snapshots,
	// archiver flushes pending job archives.
	storeStart := time.Now()
	done := make(chan struct{})
	go func() {
		defer close(done)
		var wg sync.WaitGroup

		if ms := metricstore.GetMemoryStore(); ms != nil {
			wg.Go(func() {
				metricstore.Shutdown()
			})
		}

		wg.Go(func() {
			if err := archiver.Shutdown(10 * time.Second); err != nil {
				cclog.Warnf("Archiver shutdown: %v", err)
			}
		})

		wg.Wait()
	}()

	select {
	case <-done:
		cclog.Infof("Shutdown: metricstore + archiver completed (%v)", time.Since(storeStart))
	case <-time.After(60 * time.Second):
		cclog.Warnf("Shutdown deadline exceeded after %v, forcing exit", time.Since(shutdownStart))
	}

	cclog.Infof("Shutdown: total time %v", time.Since(shutdownStart))
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go has been updated successfully.

func (s *Server) Shutdown(ctx context.Context) {
	shutdownStart := time.Now()

	natsStart := time.Now()
	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}
	cclog.Infof("Shutdown: NATS closed (%v)", time.Since(natsStart))

	httpStart := time.Now()
	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()
	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}
	cclog.Infof("Shutdown: HTTP server stopped (%v)", time.Since(httpStart))

	// Run metricstore and archiver shutdown concurrently.
	// They are independent: metricstore writes .bin snapshots,
	// archiver flushes pending job archives.
	storeStart := time.Now()
	done := make(chan struct{})
	go func() {
		defer close(done)
		var wg sync.WaitGroup

		if ms := metricstore.GetMemoryStore(); ms != nil {
			wg.Go(func() {
				metricstore.Shutdown()
			})
		}

		wg.Go(func() {
			if err := archiver.Shutdown(10 * time.Second); err != nil {
				cclog.Warnf("Archiver shutdown: %v", err)
			}
		})

		wg.Wait()
	}()

	select {
	case <-done:
		cclog.Infof("Shutdown: metricstore + archiver completed (%v)", time.Since(storeStart))
	case <-time.After(60 * time.Second):
		cclog.Warnf("Shutdown deadline exceeded after %v, forcing exit", time.Since(shutdownStart))
	}

	cclog.Infof("Shutdown: total time %v", time.Since(shutdownStart))
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package main provides the entry point for the ClusterCockpit backend server.
// This file contains HTTP server setup, routing configuration, and
// authentication middleware integration.
package main

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/99designs/gqlgen/graphql"
	"github.com/99designs/gqlgen/graphql/handler"
	"github.com/99designs/gqlgen/graphql/handler/transport"
	"github.com/99designs/gqlgen/graphql/playground"
	"github.com/ClusterCockpit/cc-backend/internal/api"
	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/auth"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/graph"
	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	"github.com/ClusterCockpit/cc-backend/web"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/runtime"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/cors"
	httpSwagger "github.com/swaggo/http-swagger"
)

var buildInfo web.Build

// Environment variable names
const (
	envDebug = "DEBUG"
)

// Server encapsulates the HTTP server state and dependencies
type Server struct {
	router        chi.Router
	server        *http.Server
	restAPIHandle *api.RestAPI
	natsAPIHandle *api.NatsAPI
}

func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) {
	rw.Header().Add("Content-Type", "application/json")
	rw.WriteHeader(http.StatusUnauthorized)
	json.NewEncoder(rw).Encode(map[string]string{
		"status": http.StatusText(http.StatusUnauthorized),
		"error":  err.Error(),
	})
}

// NewServer creates and initializes a new Server instance
func NewServer(version, commit, buildDate string) (*Server, error) {
	buildInfo = web.Build{Version: version, Hash: commit, Buildtime: buildDate}

	s := &Server{
		router: chi.NewRouter(),
	}

	if err := s.init(); err != nil {
		return nil, err
	}

	return s, nil
}

func (s *Server) init() error {
	// Setup the http.Handler/Router used by the server
	graph.Init()
	resolver := graph.GetResolverInstance()
	graphQLServer := handler.New(
		generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))

	graphQLServer.AddTransport(transport.POST{})

	// Inject a per-request stats cache so that grouped statistics queries
	// sharing the same (filter, groupBy) pair are executed only once.
	graphQLServer.AroundOperations(func(ctx context.Context, next graphql.OperationHandler) graphql.ResponseHandler {
		return next(graph.WithStatsGroupCache(ctx))
	})

	if os.Getenv(envDebug) != "1" {
		// Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed.
		// The problem with this is that then, no more stacktrace is printed to stderr.
		graphQLServer.SetRecoverFunc(func(ctx context.Context, err any) error {
			switch e := err.(type) {
			case string:
				return fmt.Errorf("MAIN > Panic: %s", e)
			case error:
				return fmt.Errorf("MAIN > Panic caused by: %s", e.Error())
			}

			return errors.New("MAIN > Internal server error (panic)")
		})
	}

	authHandle := auth.GetAuthInstance()

	// Middleware must be defined before routes in chi
	s.router.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			start := time.Now()
			ww := middleware.NewWrapResponseWriter(rw, r.ProtoMajor)
			next.ServeHTTP(ww, r)
			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
				r.Method, r.URL.RequestURI(),
				ww.Status(), float32(ww.BytesWritten())/1024,
				time.Since(start).Milliseconds())
		})
	})
	s.router.Use(middleware.Compress(5))
	s.router.Use(middleware.Recoverer)
	s.router.Use(cors.Handler(cors.Options{
		AllowCredentials: true,
		AllowedHeaders:   []string{"X-Requested-With", "Content-Type", "Authorization", "Origin"},
		AllowedMethods:   []string{"GET", "POST", "HEAD", "OPTIONS"},
		AllowedOrigins:   []string{"*"},
	}))

	s.restAPIHandle = api.New()

	info := map[string]any{}
	info["hasOpenIDConnect"] = false

	if auth.Keys.OpenIDConfig != nil {
		openIDConnect := auth.NewOIDC(authHandle)
		openIDConnect.RegisterEndpoints(s.router)
		info["hasOpenIDConnect"] = true
	}

	s.router.Get("/login", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		cclog.Debugf("##%v##", info)
		web.RenderTemplate(rw, "login.tmpl", &web.Page{Title: "Login", Build: buildInfo, Infos: info})
	})
	s.router.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		web.RenderTemplate(rw, "imprint.tmpl", &web.Page{Title: "Imprint", Build: buildInfo})
	})
	s.router.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		web.RenderTemplate(rw, "privacy.tmpl", &web.Page{Title: "Privacy", Build: buildInfo})
	})

	if !config.Keys.DisableAuthentication {
		// Create login failure handler (used by both /login and /jwt-login)
		loginFailureHandler := func(rw http.ResponseWriter, r *http.Request, err error) {
			rw.Header().Add("Content-Type", "text/html; charset=utf-8")
			rw.WriteHeader(http.StatusUnauthorized)
			web.RenderTemplate(rw, "login.tmpl", &web.Page{
				Title:   "Login failed - ClusterCockpit",
				MsgType: "alert-warning",
				Message: err.Error(),
				Build:   buildInfo,
				Infos:   info,
			})
		}

		s.router.Post("/login", authHandle.Login(loginFailureHandler).ServeHTTP)
		s.router.HandleFunc("/jwt-login", authHandle.Login(loginFailureHandler).ServeHTTP)

		s.router.Post("/logout", authHandle.Logout(
			http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
				rw.WriteHeader(http.StatusOK)
				web.RenderTemplate(rw, "login.tmpl", &web.Page{
					Title:   "Bye - ClusterCockpit",
					MsgType: "alert-info",
					Message: "Logout successful",
					Build:   buildInfo,
					Infos:   info,
				})
			})).ServeHTTP)
	}

	if flagDev {
		s.router.Handle("/playground", playground.Handler("GraphQL playground", "/query"))
		s.router.Get("/swagger/*", httpSwagger.Handler(
			httpSwagger.URL("http://"+config.Keys.Addr+"/swagger/doc.json")))
	}

	// Secured routes (require authentication)
	s.router.Group(func(secured chi.Router) {
		if !config.Keys.DisableAuthentication {
			secured.Use(func(next http.Handler) http.Handler {
				return authHandle.Auth(
					next,
					func(rw http.ResponseWriter, r *http.Request, err error) {
						rw.WriteHeader(http.StatusUnauthorized)
						web.RenderTemplate(rw, "login.tmpl", &web.Page{
							Title:    "Authentication failed - ClusterCockpit",
							MsgType:  "alert-danger",
							Message:  err.Error(),
							Build:    buildInfo,
							Infos:    info,
							Redirect: r.RequestURI,
						})
					})
			})
		}

		secured.Handle("/query", graphQLServer)

		secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
			routerConfig.HandleSearchBar(rw, r, buildInfo)
		})

		routerConfig.SetupRoutes(secured, buildInfo)
	})

	// API routes (JWT token auth)
	s.router.Route("/api", func(apiRouter chi.Router) {
		// Main API routes with API auth
		apiRouter.Group(func(securedapi chi.Router) {
			if !config.Keys.DisableAuthentication {
				securedapi.Use(func(next http.Handler) http.Handler {
					return authHandle.AuthAPI(next, onFailureResponse)
				})
			}
			s.restAPIHandle.MountAPIRoutes(securedapi)
		})

		// Metric store API routes with separate auth
		apiRouter.Group(func(metricstoreapi chi.Router) {
			if !config.Keys.DisableAuthentication {
				metricstoreapi.Use(func(next http.Handler) http.Handler {
					return authHandle.AuthMetricStoreAPI(next, onFailureResponse)
				})
			}
			s.restAPIHandle.MountMetricStoreAPIRoutes(metricstoreapi)
		})
	})

	// User API routes
	s.router.Route("/userapi", func(userapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			userapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthUserAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountUserAPIRoutes(userapi)
	})

	// Config API routes (uses Group with full paths to avoid shadowing
	// the /config page route that is registered in the secured group)
	s.router.Group(func(configapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			configapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthConfigAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountConfigAPIRoutes(configapi)
	})

	// Frontend API routes
	s.router.Route("/frontend", func(frontendapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			frontendapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthFrontendAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountFrontendAPIRoutes(frontendapi)
	})

	if config.Keys.APISubjects != nil {
		s.natsAPIHandle = api.NewNatsAPI()
		if err := s.natsAPIHandle.StartSubscriptions(); err != nil {
			return fmt.Errorf("starting NATS subscriptions: %w", err)
		}
	}

	// 404 handler for pages and API routes
	notFoundHandler := func(rw http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/userapi/") ||
			strings.HasPrefix(r.URL.Path, "/frontend/") || strings.HasPrefix(r.URL.Path, "/config/") {
			rw.Header().Set("Content-Type", "application/json")
			rw.WriteHeader(http.StatusNotFound)
			json.NewEncoder(rw).Encode(map[string]string{
				"status": "Resource not found",
				"error":  "the requested endpoint does not exist",
			})
			return
		}
		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
		rw.WriteHeader(http.StatusNotFound)
		web.RenderTemplate(rw, "404.tmpl", &web.Page{
			Title: "Page Not Found",
			Build: buildInfo,
		})
	}

	// Set NotFound on the router so chi uses it for all unmatched routes,
	// including those under subrouters like /api, /userapi, /frontend, etc.
	s.router.NotFound(notFoundHandler)

	if config.Keys.EmbedStaticFiles {
		if i, err := os.Stat("./var/img"); err == nil {
			if i.IsDir() {
				cclog.Info("Use local directory for static images")
				s.router.Handle("/img/*", http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
			}
		}
		fileServer := http.StripPrefix("/", web.ServeFiles())
		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			if web.StaticFileExists(r.URL.Path) {
				fileServer.ServeHTTP(rw, r)
				return
			}
			notFoundHandler(rw, r)
		}))
	} else {
		staticDir := http.Dir(config.Keys.StaticFiles)
		fileServer := http.FileServer(staticDir)
		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			f, err := staticDir.Open(r.URL.Path)
			if err == nil {
				f.Close()
				fileServer.ServeHTTP(rw, r)
				return
			}
			notFoundHandler(rw, r)
		}))
	}

	return nil
}

// Server timeout defaults (in seconds)
const (
	defaultReadHeaderTimeout = 20
	defaultWriteTimeout      = 20
)

func (s *Server) Start(ctx context.Context) error {
	// Use configurable timeouts with defaults
	readHeaderTimeout := time.Duration(defaultReadHeaderTimeout) * time.Second
	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second

	s.server = &http.Server{
		ReadHeaderTimeout: readHeaderTimeout,
		WriteTimeout:      writeTimeout,
		Handler:           s.router,
		Addr:              config.Keys.Addr,
	}

	// Start http or https server
	listener, err := net.Listen("tcp", config.Keys.Addr)
	if err != nil {
		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Addr, err)
	}

	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHTTPTo != "" {
		go func() {
			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHTTPTo, http.StatusMovedPermanently))
		}()
	}

	if config.Keys.HTTPSCertFile != "" && config.Keys.HTTPSKeyFile != "" {
		cert, err := tls.LoadX509KeyPair(
			config.Keys.HTTPSCertFile, config.Keys.HTTPSKeyFile)
		if err != nil {
			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
		}
		listener = tls.NewListener(listener, &tls.Config{
			Certificates: []tls.Certificate{cert},
			CipherSuites: []uint16{
				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			},
			MinVersion:               tls.VersionTLS12,
			PreferServerCipherSuites: true,
		})
		cclog.Infof("HTTPS server listening at %s...", config.Keys.Addr)
	} else {
		cclog.Infof("HTTP server listening at %s...", config.Keys.Addr)
	}
	//
	// Because this program will want to bind to a privileged port (like 80), the listener must
	// be established first, then the user can be changed, and after that,
	// the actual http server can be started.
	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
		return fmt.Errorf("dropping privileges: %w", err)
	}

	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
		return fmt.Errorf("server failed: %w", err)
	}
	return nil
}

func (s *Server) Shutdown(ctx context.Context) {
	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Run metricstore and archiver shutdown concurrently.
	// They are independent: metricstore writes .bin snapshots,
	// archiver flushes pending job archives.
	done := make(chan struct{})
	go func() {
		defer close(done)
		var wg sync.WaitGroup

		if ms := metricstore.GetMemoryStore(); ms != nil {
			wg.Go(func() {
				metricstore.Shutdown()
			})
		}

		wg.Go(func() {
			if err := archiver.Shutdown(10 * time.Second); err != nil {
				cclog.Warnf("Archiver shutdown: %v", err)
			}
		})

		wg.Wait()
	}()

	select {
	case <-done:
	case <-time.After(60 * time.Second):
		cclog.Warn("Shutdown deadline exceeded, forcing exit")
	}
}

-  shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)

+  cclog.Infof("Shutdown: NATS closed (%v)", time.Since(natsStart))

+  shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)

+  cclog.Infof("Shutdown: HTTP server stopped (%v)", time.Since(httpStart))

+    cclog.Infof("Shutdown: metricstore + archiver completed (%v)", time.Since(storeStart))

-    cclog.Warn("Shutdown deadline exceeded, forcing exit")

+    cclog.Warnf("Shutdown deadline exceeded after %v, forcing exit", time.Since(shutdownStart))

+  cclog.Infof("Shutdown: total time %v", time.Since(shutdownStart))


Updated task #5 status


Bash

Build all packages


(Bash completed with no output)


Run metricstore tests


ok  	github.com/ClusterCockpit/cc-backend/pkg/metricstore	0.650s


go vet ./pkg/metricstore/... ./cmd/cc-backend/... 2>&1

Run vet on changed packages


All changes compile, pass tests, and pass vet. Here's a summary of what was done:

**Root cause**: WAL sharding buffers up to 65,536 messages. On shutdown, `drain()` processed every buffered message with file I/O before checkpoint writes could start — wasting the entire 60s deadline on redundant work (since `ToCheckpointWAL` writes a full snapshot from in-memory state anyway).

**Changes made across 3 files:**

1. **`pkg/metricstore/walCheckpoint.go`**:
   - `ctx.Done()` handler now calls `flushDirty()` instead of `drain()` — skips processing redundant buffered messages
   - Added `walStagingWg` + `WaitForWALStagingDrain()` for synchronization before checkpoint writes
   - `ToCheckpointWAL()` now deletes each host's `current.wal` immediately after successful snapshot (not all at the end)
   - Added progress logging every 10s with host count, written/error counts

2. **`pkg/metricstore/metricstore.go`**:
   - `Shutdown()` waits for WAL staging goroutines to exit before checkpointing
   - Removed `RotateWALFilesAfterShutdown()` call (WAL deletion moved into workers)
   - Added timing logs for each shutdown phase

3. **`cmd/cc-backend/server.go`**:
   - Added timing logs for NATS close, HTTP stop, metricstore+archiver, and total shutdown time


Do an in-depth review of the sharded WAL buffer pipeline. Including startup and shutdown. Find potential problems and suggest imrovements.


Agent

Explore WAL pipeline end-to-end

Prompt:
I need a comprehensive review of the sharded WAL buffer pipeline in the cc-backend metricstore. Read ALL relevant code thoroughly - every function, every goroutine, every channel interaction. I need to find bugs, races, and improvement opportunities.

Read these files COMPLETELY (not just excerpts):

1. `pkg/metricstore/walCheckpoint.go` — the entire file, all functions
2. `pkg/metricstore/lineprotocol.go` — how WAL messages are sent (the producer side)
3. `pkg/metricstore/metricstore.go` — Init(), Shutdown(), and all related setup
4. `pkg/metricstore/checkpoint.go` — the Checkpointing goroutine and periodic checkpoint flow

For each file, read it in chunks to get the complete picture. Pay special attention to:

- Channel lifecycle: creation, buffering, closing, draining
- Goroutine lifecycle: startup, shutdown, context cancellation
- Race conditions between producers and consumers
- Error handling and recovery
- What happens when channels are full (backpressure)
- The interaction between periodic checkpointing and WAL rotation
- Startup: how WAL files are loaded/replayed
- Memory allocation patterns in hot paths
- Lock ordering and potential deadlocks
- What happens if a WAL write fails mid-record
- The walRotateReq mechanism and its interaction with shutdown

Be very thorough. Read every line.


I need a comprehensive review of the sharded WAL buffer pipeline in the cc-backend metricstore. Read ALL relevant code thoroughly - every function, every goroutine, every channel interaction. I need to find bugs, races, and improvement opportunities.

Read these files COMPLETELY (not just excerpts):

1. `pkg/metricstore/walCheckpoint.go` — the entire file, all functions
2. `pkg/metricstore/lineprotocol.go` — how WAL messages are sent (the producer side)
3. `pkg/metricstore/metricstore.go` — Init(), Shutdown(), and all related setup
4. `pkg/metricstore/checkpoint.go` — the Checkpointing goroutine and periodic checkpoint flow

For each file, read it in chunks to get the complete picture. Pay special attention to:

- Channel lifecycle: creation, buffering, closing, draining
- Goroutine lifecycle: startup, shutdown, context cancellation
- Race conditions between producers and consumers
- Error handling and recovery
- What happens when channels are full (backpressure)
- The interaction between periodic checkpointing and WAL rotation
- Startup: how WAL files are loaded/replayed
- Memory allocation patterns in hot paths
- Lock ordering and potential deadlocks
- What happens if a WAL write fails mid-record
- The walRotateReq mechanism and its interaction with shutdown

Be very thorough. Read every line.

Prompt:
I need a comprehensive review of the sharded WAL buffer pipeline in the cc-backend metricstore. Read ALL relevant code thoroughly - every function, every goroutine, every channel interaction. I need to find bugs, races, and improvement opportunities.

Read these files COMPLETELY (not just excerpts):

1. `pkg/metricstore/walCheckpoint.go` — the entire file, all functions
2. `pkg/metricstore/lineprotocol.go` — how WAL messages are sent (the producer side)
3. `pkg/metricstore/metricstore.go` — Init(), Shutdown(), and all related setup
4. `pkg/metricstore/checkpoint.go` — the Checkpointing goroutine and periodic checkpoint flow

For each file, read it in chunks to get the complete picture. Pay special attention to:

- Channel lifecycle: creation, buffering, closing, draining
- Goroutine lifecycle: startup, shutdown, context cancellation
- Race conditions between producers and consumers
- Error handling and recovery
- What happens when channels are full (backpressure)
- The interaction between periodic checkpointing and WAL rotation
- Startup: how WAL files are loaded/replayed
- Memory allocation patterns in hot paths
- Lock ordering and potential deadlocks
- What happens if a WAL write fails mid-record
- The walRotateReq mechanism and its interaction with shutdown

Be very thorough. Read every line.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
     7→//
     8→// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
     9→// per host, eliminating the extra memory overhead of the AvroStore and providing
    10→// truly continuous (per-write) crash safety.
    11→//
    12→// # Architecture
    13→//
    14→//	Metric write (DecodeLine)
    15→//	      │
    16→//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
    17→//	      │
    18→//	      └─► WALMessages channel
    19→//	               │
    20→//	               ▼
    21→//	        WALStaging goroutine
    22→//	               │
    23→//	               ▼
    24→//	        checkpoints/cluster/host/current.wal  (append-only, binary)
    25→//
    26→//	Periodic checkpoint (Checkpointing goroutine):
    27→//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
    28→//	  2. Signal WALStaging to truncate current.wal per host
    29→//
    30→//	On restart (FromCheckpoint):
    31→//	  1. Load most recent <timestamp>.bin snapshot
    32→//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
    33→//
    34→// # WAL Record Format
    35→//
    36→//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
    37→//
    38→//	payload:
    39→//	  [8B timestamp int64]
    40→//	  [2B metric_name_len uint16][N metric name bytes]
    41→//	  [1B selector_count uint8]
    42→//	  per selector: [1B selector_len uint8][M selector bytes]
    43→//	  [4B value float32 bits]
    44→//
    45→// # Binary Snapshot Format
    46→//
    47→//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
    48→//	Level tree (recursive):
    49→//	  [4B num_metrics uint32]
    50→//	  per metric:
    51→//	    [2B name_len uint16][N name bytes]
    52→//	    [8B frequency int64][8B start int64]
    53→//	    [4B num_values uint32][num_values × 4B float32]
    54→//	  [4B num_children uint32]
    55→//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
    56→package metricstore
    57→
    58→import (
    59→	"bufio"
    60→	"context"
    61→	"encoding/binary"
    62→	"fmt"
    63→	"hash/crc32"
    64→	"hash/fnv"
    65→	"io"
    66→	"math"
    67→	"os"
    68→	"path"
    69→	"strings"
    70→	"sync"
    71→	"sync/atomic"
    72→	"time"
    73→
    74→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    75→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    76→)
    77→
    78→// Magic numbers for binary formats.
    79→const (
    80→	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
    81→	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
    82→	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
    83→)
    84→
    85→// walShardChs holds per-shard channels for WAL messages.
    86→// Initialized by WALStaging; nil when WAL is not active.
    87→var walShardChs []chan *WALMessage
    88→
    89→// walShardRotateChs holds per-shard channels for WAL rotation requests.
    90→var walShardRotateChs []chan walRotateReq
    91→
    92→// walNumShards stores the number of shards (set during WALStaging init).
    93→var walNumShards int
    94→
    95→// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
    96→var walStagingWg sync.WaitGroup
    97→
    98→// WALMessage represents a single metric write to be appended to the WAL.
    99→// Cluster and Node are NOT stored in the WAL record (inferred from file path).
   100→type WALMessage struct {
   101→	MetricName string
   102→	Cluster    string
   103→	Node       string
   104→	Selector   []string
   105→	Value      schema.Float
   106→	Timestamp  int64
   107→}
   108→
   109→// walRotateReq requests WAL file rotation for a specific host directory.
   110→// The done channel is closed by the WAL goroutine when rotation is complete.
   111→type walRotateReq struct {
   112→	hostDir string
   113→	done    chan struct{}
   114→}
   115→
   116→// walFileState holds an open WAL file handle and buffered writer for one host directory.
   117→type walFileState struct {
   118→	f     *os.File
   119→	w     *bufio.Writer
   120→	dirty bool
   121→}
   122→
   123→// walFlushInterval controls how often dirty WAL files are flushed to disk.
   124→// Decoupling flushes from message processing lets the consumer run at memory
   125→// speed, amortizing syscall overhead across many writes.
   126→const walFlushInterval = 5 * time.Second
   127→
   128→// walShardIndex computes which shard a message belongs to based on cluster+node.
   129→// Uses FNV-1a hash for fast, well-distributed mapping.
   130→func walShardIndex(cluster, node string) int {
   131→	h := fnv.New32a()
   132→	h.Write([]byte(cluster))
   133→	h.Write([]byte{0})
   134→	h.Write([]byte(node))
   135→	return int(h.Sum32()) % walNumShards
   136→}
   137→
   138→// SendWALMessage routes a WAL message to the appropriate shard channel.
   139→// Returns false if the channel is full (message dropped).
   140→func SendWALMessage(msg *WALMessage) bool {
   141→	if walShardChs == nil {
   142→		return false
   143→	}
   144→	shard := walShardIndex(msg.Cluster, msg.Node)
   145→	select {
   146→	case walShardChs[shard] <- msg:
   147→		return true
   148→	default:
   149→		return false
   150→	}
   151→}
   152→
   153→// WALStaging starts N sharded background goroutines that receive WALMessage items
   154→// and append binary WAL records to per-host current.wal files.
   155→// Each shard owns a subset of hosts (determined by hash of cluster+node),
   156→// parallelizing serialization and I/O while keeping per-host writes serial.
   157→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   158→	if Keys.Checkpoints.FileFormat == "json" {
   159→		return
   160→	}
   161→
   162→	walNumShards = max(Keys.NumWorkers, 1)
   163→	chBufSize := max(65536/walNumShards, 1024)
   164→
   165→	walShardChs = make([]chan *WALMessage, walNumShards)
   166→	walShardRotateChs = make([]chan walRotateReq, walNumShards)
   167→
   168→	for i := range walNumShards {
   169→		walShardChs[i] = make(chan *WALMessage, chBufSize)
   170→		walShardRotateChs[i] = make(chan walRotateReq, 64)
   171→	}
   172→
   173→	for i := range walNumShards {
   174→		msgCh := walShardChs[i]
   175→		rotateCh := walShardRotateChs[i]
   176→
   177→		walStagingWg.Add(1)
   178→		wg.Go(func() {
   179→			defer walStagingWg.Done()
   180→			hostFiles := make(map[string]*walFileState)
   181→
   182→			defer func() {
   183→				for _, ws := range hostFiles {
   184→					if ws.f != nil {
   185→						ws.w.Flush()
   186→						ws.f.Close()
   187→					}
   188→				}
   189→			}()
   190→
   191→			getOrOpenWAL := func(hostDir string) *walFileState {
   192→				ws, ok := hostFiles[hostDir]
   193→				if ok {
   194→					return ws
   195→				}
   196→
   197→				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   198→					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   199→					return nil
   200→				}
   201→
   202→				walPath := path.Join(hostDir, "current.wal")
   203→				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   204→				if err != nil {
   205→					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   206→					return nil
   207→				}
   208→
   209→				w := bufio.NewWriter(f)
   210→
   211→				// Write file header magic if file is new (empty).
   212→				info, err := f.Stat()
   213→				if err == nil && info.Size() == 0 {
   214→					var hdr [4]byte
   215→					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   216→					if _, err := w.Write(hdr[:]); err != nil {
   217→						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   218→						f.Close()
   219→						return nil
   220→					}
   221→				}
   222→
   223→				ws = &walFileState{f: f, w: w}
   224→				hostFiles[hostDir] = ws
   225→				return ws
   226→			}
   227→
   228→			processMsg := func(msg *WALMessage) {
   229→				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   230→				ws := getOrOpenWAL(hostDir)
   231→				if ws == nil {
   232→					return
   233→				}
   234→				if err := writeWALRecordDirect(ws.w, msg); err != nil {
   235→					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   236→				}
   237→				ws.dirty = true
   238→			}
   239→
   240→			processRotate := func(req walRotateReq) {
   241→				ws, ok := hostFiles[req.hostDir]
   242→				if ok && ws.f != nil {
   243→					ws.w.Flush()
   244→					ws.f.Close()
   245→					walPath := path.Join(req.hostDir, "current.wal")
   246→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   247→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   248→					}
   249→					delete(hostFiles, req.hostDir)
   250→				}
   251→				close(req.done)
   252→			}
   253→
   254→			flushDirty := func() {
   255→				for _, ws := range hostFiles {
   256→					if ws.dirty {
   257→						ws.w.Flush()
   258→						ws.dirty = false
   259→					}
   260→				}
   261→			}
   262→
   263→			ticker := time.NewTicker(walFlushInterval)
   264→			defer ticker.Stop()
   265→
   266→			// drainBatch processes up to 4096 pending messages without blocking.
   267→			// Returns false if the channel was closed.
   268→			drainBatch := func() bool {
   269→				for range 4096 {
   270→					select {
   271→					case msg, ok := <-msgCh:
   272→						if !ok {
   273→							flushDirty()
   274→							return false
   275→						}
   276→						processMsg(msg)
   277→					case req := <-rotateCh:
   278→						processRotate(req)
   279→					default:
   280→						return true
   281→					}
   282→				}
   283→				return true
   284→			}
   285→
   286→			for {
   287→				select {
   288→				case <-ctx.Done():
   289→					// On shutdown, skip draining buffered messages — a full binary
   290→					// checkpoint will be written from in-memory state, making
   291→					// buffered WAL records redundant.
   292→					flushDirty()
   293→					return
   294→				case msg, ok := <-msgCh:
   295→					if !ok {
   296→						return
   297→					}
   298→					processMsg(msg)
   299→					if !drainBatch() {
   300→						return
   301→					}
   302→					// No flush here — timer handles periodic flushing.
   303→				case <-ticker.C:
   304→					flushDirty()
   305→				case req := <-rotateCh:
   306→					processRotate(req)
   307→				}
   308→			}
   309→		})
   310→	}
   311→}
   312→
   313→// WaitForWALStagingDrain blocks until all WALStaging goroutines have exited.
   314→// Must be called after closing walShardChs to ensure all file handles are
   315→// flushed and closed before checkpoint writes begin.
   316→func WaitForWALStagingDrain() {
   317→	walStagingWg.Wait()
   318→}
   319→
   320→// RotateWALFiles sends rotation requests for the given host directories
   321→// and blocks until all rotations complete. Each request is routed to the
   322→// shard that owns the host directory.
   323→func RotateWALFiles(hostDirs []string) {
   324→	if walShardRotateChs == nil {
   325→		return
   326→	}
   327→	dones := make([]chan struct{}, len(hostDirs))
   328→	for i, dir := range hostDirs {
   329→		dones[i] = make(chan struct{})
   330→		// Extract cluster/node from hostDir to find the right shard.
   331→		// hostDir = rootDir/cluster/node
   332→		shard := walShardIndexFromDir(dir)
   333→		walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: dones[i]}
   334→	}
   335→	for _, done := range dones {
   336→		<-done
   337→	}
   338→}
   339→
   340→// walShardIndexFromDir extracts cluster and node from a host directory path
   341→// and returns the shard index. The path format is: rootDir/cluster/node
   342→func walShardIndexFromDir(hostDir string) int {
   343→	// hostDir = .../cluster/node — extract last two path components
   344→	node := path.Base(hostDir)
   345→	cluster := path.Base(path.Dir(hostDir))
   346→	return walShardIndex(cluster, node)
   347→}
   348→
   349→// RotateWALFiles sends rotation requests for the given host directories
   350→// and blocks until all rotations complete.
   351→func RotateWALFilesAfterShutdown(hostDirs []string) {
   352→	for _, dir := range hostDirs {
   353→		walPath := path.Join(dir, "current.wal")
   354→		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   355→			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   356→		}
   357→	}
   358→}
   359→
   360→// writeWALRecordDirect encodes a WAL record directly into the bufio.Writer,
   361→// avoiding heap allocations by using a stack-allocated scratch buffer for
   362→// the fixed-size header/trailer and computing CRC inline.
   363→func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
   364→	// Compute payload size.
   365→	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
   366→	for _, s := range msg.Selector {
   367→		payloadSize += 1 + len(s)
   368→	}
   369→
   370→	// Write magic + payload length (8 bytes header).
   371→	var hdr [8]byte
   372→	binary.LittleEndian.PutUint32(hdr[0:4], walRecordMagic)
   373→	binary.LittleEndian.PutUint32(hdr[4:8], uint32(payloadSize))
   374→	if _, err := w.Write(hdr[:]); err != nil {
   375→		return err
   376→	}
   377→
   378→	// We need to compute CRC over the payload as we write it.
   379→	crc := crc32.NewIEEE()
   380→
   381→	// Timestamp (8 bytes).
   382→	var scratch [8]byte
   383→	binary.LittleEndian.PutUint64(scratch[:8], uint64(msg.Timestamp))
   384→	crc.Write(scratch[:8])
   385→	if _, err := w.Write(scratch[:8]); err != nil {
   386→		return err
   387→	}
   388→
   389→	// Metric name length (2 bytes) + metric name.
   390→	binary.LittleEndian.PutUint16(scratch[:2], uint16(len(msg.MetricName)))
   391→	crc.Write(scratch[:2])
   392→	if _, err := w.Write(scratch[:2]); err != nil {
   393→		return err
   394→	}
   395→	nameBytes := []byte(msg.MetricName)
   396→	crc.Write(nameBytes)
   397→	if _, err := w.Write(nameBytes); err != nil {
   398→		return err
   399→	}
   400→
   401→	// Selector count (1 byte).
   402→	scratch[0] = byte(len(msg.Selector))
   403→	crc.Write(scratch[:1])
   404→	if _, err := w.Write(scratch[:1]); err != nil {
   405→		return err
   406→	}
   407→
   408→	// Selectors (1-byte length + bytes each).
   409→	for _, sel := range msg.Selector {
   410→		scratch[0] = byte(len(sel))
   411→		crc.Write(scratch[:1])
   412→		if _, err := w.Write(scratch[:1]); err != nil {
   413→			return err
   414→		}
   415→		selBytes := []byte(sel)
   416→		crc.Write(selBytes)
   417→		if _, err := w.Write(selBytes); err != nil {
   418→			return err
   419→		}
   420→	}
   421→
   422→	// Value (4 bytes, float32 bits).
   423→	binary.LittleEndian.PutUint32(scratch[:4], math.Float32bits(float32(msg.Value)))
   424→	crc.Write(scratch[:4])
   425→	if _, err := w.Write(scratch[:4]); err != nil {
   426→		return err
   427→	}
   428→
   429→	// CRC32 (4 bytes).
   430→	binary.LittleEndian.PutUint32(scratch[:4], crc.Sum32())
   431→	_, err := w.Write(scratch[:4])
   432→	return err
   433→}
   434→
   435→// readWALRecord reads one WAL record from the reader.
   436→// Returns (nil, nil) on clean EOF. Returns error on data corruption.
   437→// A CRC mismatch indicates a truncated trailing record (expected on crash).
   438→func readWALRecord(r io.Reader) (*WALMessage, error) {
   439→	var magic uint32
   440→	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
   441→		if err == io.EOF {
   442→			return nil, nil // Clean EOF
   443→		}
   444→		return nil, fmt.Errorf("read record magic: %w", err)
   445→	}
   446→
   447→	if magic != walRecordMagic {
   448→		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
   449→	}
   450→
   451→	var payloadLen uint32
   452→	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
   453→		return nil, fmt.Errorf("read payload length: %w", err)
   454→	}
   455→
   456→	if payloadLen > 1<<20 { // 1 MB sanity limit
   457→		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
   458→	}
   459→
   460→	payload := make([]byte, payloadLen)
   461→	if _, err := io.ReadFull(r, payload); err != nil {
   462→		return nil, fmt.Errorf("read payload: %w", err)
   463→	}
   464→
   465→	var storedCRC uint32
   466→	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
   467→		return nil, fmt.Errorf("read CRC: %w", err)
   468→	}
   469→
   470→	if crc32.ChecksumIEEE(payload) != storedCRC {
   471→		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
   472→	}
   473→
   474→	return parseWALPayload(payload)
   475→}
   476→
   477→// parseWALPayload decodes a binary payload into a WALMessage.
   478→func parseWALPayload(payload []byte) (*WALMessage, error) {
   479→	if len(payload) < 8+2+1+4 {
   480→		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
   481→	}
   482→
   483→	offset := 0
   484→
   485→	// Timestamp (8 bytes)
   486→	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
   487→	offset += 8
   488→
   489→	// Metric name (2-byte length + bytes)
   490→	if offset+2 > len(payload) {
   491→		return nil, fmt.Errorf("metric name length overflows payload")
   492→	}
   493→	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
   494→	offset += 2
   495→
   496→	if offset+mLen > len(payload) {
   497→		return nil, fmt.Errorf("metric name overflows payload")
   498→	}
   499→	metricName := string(payload[offset : offset+mLen])
   500→	offset += mLen
   501→
   502→	// Selector count (1 byte)
   503→	if offset >= len(payload) {
   504→		return nil, fmt.Errorf("selector count overflows payload")
   505→	}
   506→	selCount := int(payload[offset])
   507→	offset++
   508→
   509→	selectors := make([]string, selCount)
   510→	for i := range selCount {
   511→		if offset >= len(payload) {
   512→			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
   513→		}
   514→		sLen := int(payload[offset])
   515→		offset++
   516→
   517→		if offset+sLen > len(payload) {
   518→			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
   519→		}
   520→		selectors[i] = string(payload[offset : offset+sLen])
   521→		offset += sLen
   522→	}
   523→
   524→	// Value (4 bytes, float32 bits)
   525→	if offset+4 > len(payload) {
   526→		return nil, fmt.Errorf("value overflows payload")
   527→	}
   528→	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
   529→	value := schema.Float(math.Float32frombits(bits))
   530→
   531→	return &WALMessage{
   532→		MetricName: metricName,
   533→		Timestamp:  ts,
   534→		Selector:   selectors,
   535→		Value:      value,
   536→	}, nil
   537→}
   538→
   539→// loadWALFile reads a WAL file and replays all valid records into the Level tree.
   540→// l is the host-level node. Corrupt or partial trailing records are silently skipped
   541→// (expected on crash). Records older than 'from' are skipped.
   542→func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
   543→	br := bufio.NewReader(f)
   544→
   545→	// Verify file header magic.
   546→	var fileMagic uint32
   547→	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
   548→		if err == io.EOF {
   549→			return nil // Empty file, no data
   550→		}
   551→		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
   552→	}
   553→
   554→	if fileMagic != walFileMagic {
   555→		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
   556→	}
   557→
   558→	// Cache level lookups to avoid repeated tree traversal.
   559→	lvlCache := make(map[string]*Level)
   560→
   561→	for {
   562→		msg, err := readWALRecord(br)
   563→		if err != nil {
   564→			// Truncated trailing record is expected after a crash; stop replaying.
   565→			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
   566→			break
   567→		}
   568→		if msg == nil {
   569→			break // Clean EOF
   570→		}
   571→
   572→		if msg.Timestamp < from {
   573→			continue // Older than retention window
   574→		}
   575→
   576→		minfo, ok := m.Metrics[msg.MetricName]
   577→		if !ok {
   578→			continue // Unknown metric (config may have changed)
   579→		}
   580→
   581→		// Cache key is the null-separated selector path.
   582→		cacheKey := joinSelector(msg.Selector)
   583→		lvl, ok := lvlCache[cacheKey]
   584→		if !ok {
   585→			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
   586→			lvlCache[cacheKey] = lvl
   587→		}
   588→
   589→		// Write directly to the buffer, same as WriteToLevel but without the
   590→		// global level lookup (we already have the right level).
   591→		lvl.lock.Lock()
   592→		b := lvl.metrics[minfo.offset]
   593→		if b == nil {
   594→			b = newBuffer(msg.Timestamp, minfo.Frequency)
   595→			lvl.metrics[minfo.offset] = b
   596→		}
   597→		nb, writeErr := b.write(msg.Timestamp, msg.Value)
   598→		if writeErr == nil && b != nb {
   599→			lvl.metrics[minfo.offset] = nb
   600→		}
   601→		// Ignore write errors for timestamps before buffer start (can happen when
   602→		// replaying WAL entries that predate a loaded snapshot's start time).
   603→		lvl.lock.Unlock()
   604→	}
   605→
   606→	return nil
   607→}
   608→
   609→// joinSelector builds a cache key from a selector slice using null bytes as separators.
   610→func joinSelector(sel []string) string {
   611→	if len(sel) == 0 {
   612→		return ""
   613→	}
   614→	var result strings.Builder
   615→	result.WriteString(sel[0])
   616→	for i := 1; i < len(sel); i++ {
   617→		result.WriteString("\x00" + sel[i])
   618→	}
   619→	return result.String()
   620→}
   621→
   622→// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
   623→// Returns the number of files written, the list of host directories that were
   624→// successfully checkpointed (for WAL rotation), and any errors.
   625→func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
   626→	// Collect all cluster/host pairs.
   627→	m.root.lock.RLock()
   628→	totalHosts := 0
   629→	for _, l1 := range m.root.children {
   630→		l1.lock.RLock()
   631→		totalHosts += len(l1.children)
   632→		l1.lock.RUnlock()
   633→	}
   634→	m.root.lock.RUnlock()
   635→
   636→	levels := make([]*Level, 0, totalHosts)
   637→	selectors := make([][]string, 0, totalHosts)
   638→
   639→	m.root.lock.RLock()
   640→	for sel1, l1 := range m.root.children {
   641→		l1.lock.RLock()
   642→		for sel2, l2 := range l1.children {
   643→			levels = append(levels, l2)
   644→			selectors = append(selectors, []string{sel1, sel2})
   645→		}
   646→		l1.lock.RUnlock()
   647→	}
   648→	m.root.lock.RUnlock()
   649→
   650→	type workItem struct {
   651→		level    *Level
   652→		hostDir  string
   653→		selector []string
   654→	}
   655→
   656→	totalWork := len(levels)
   657→	cclog.Infof("[METRICSTORE]> Starting binary checkpoint for %d hosts with %d workers", totalWork, Keys.NumWorkers)
   658→
   659→	n, errs, completed := int32(0), int32(0), int32(0)
   660→	var successDirs []string
   661→	var successMu sync.Mutex
   662→
   663→	var wg sync.WaitGroup
   664→	wg.Add(Keys.NumWorkers)
   665→	work := make(chan workItem, Keys.NumWorkers*2)
   666→
   667→	// Progress logging goroutine.
   668→	stopProgress := make(chan struct{})
   669→	go func() {
   670→		ticker := time.NewTicker(10 * time.Second)
   671→		defer ticker.Stop()
   672→		for {
   673→			select {
   674→			case <-ticker.C:
   675→				cclog.Infof("[METRICSTORE]> Checkpoint progress: %d/%d hosts (%d written, %d errors)",
   676→					atomic.LoadInt32(&completed), totalWork, atomic.LoadInt32(&n), atomic.LoadInt32(&errs))
   677→			case <-stopProgress:
   678→				return
   679→			}
   680→		}
   681→	}()
   682→
   683→	for range Keys.NumWorkers {
   684→		go func() {
   685→			defer wg.Done()
   686→			for wi := range work {
   687→				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
   688→				if err != nil {
   689→					if err == ErrNoNewArchiveData {
   690→						atomic.AddInt32(&completed, 1)
   691→						continue
   692→					}
   693→					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
   694→					atomic.AddInt32(&errs, 1)
   695→				} else {
   696→					atomic.AddInt32(&n, 1)
   697→					// Delete WAL immediately after successful snapshot.
   698→					walPath := path.Join(wi.hostDir, "current.wal")
   699→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   700→						cclog.Errorf("[METRICSTORE]> WAL remove %s: %v", walPath, err)
   701→					}
   702→					successMu.Lock()
   703→					successDirs = append(successDirs, wi.hostDir)
   704→					successMu.Unlock()
   705→				}
   706→				atomic.AddInt32(&completed, 1)
   707→			}
   708→		}()
   709→	}
   710→
   711→	for i := range levels {
   712→		hostDir := path.Join(dir, path.Join(selectors[i]...))
   713→		work <- workItem{
   714→			level:    levels[i],
   715→			hostDir:  hostDir,
   716→			selector: selectors[i],
   717→		}
   718→	}
   719→	close(work)
   720→	wg.Wait()
   721→	close(stopProgress)
   722→
   723→	if errs > 0 {
   724→		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
   725→	}
   726→	return int(n), successDirs, nil
   727→}
   728→
   729→// toCheckpointBinary writes a binary snapshot file for a single host-level node.
   730→// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
   731→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   732→	cf, err := l.toCheckpointFile(from, to, m)
   733→	if err != nil {
   734→		return err
   735→	}
   736→	if cf == nil {
   737→		return ErrNoNewArchiveData
   738→	}
   739→
   740→	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
   741→		return fmt.Errorf("mkdir %s: %w", dir, err)
   742→	}
   743→
   744→	// Write to a temp file first, then rename (atomic on POSIX).
   745→	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
   746→	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))
   747→
   748→	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   749→	if err != nil {
   750→		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
   751→	}
   752→
   753→	bw := bufio.NewWriter(f)
   754→	if err := writeBinarySnapshotFile(bw, cf); err != nil {
   755→		f.Close()
   756→		os.Remove(tmpPath)
   757→		return fmt.Errorf("write binary snapshot: %w", err)
   758→	}
   759→	if err := bw.Flush(); err != nil {
   760→		f.Close()
   761→		os.Remove(tmpPath)
   762→		return err
   763→	}
   764→	f.Close()
   765→
   766→	return os.Rename(tmpPath, finalPath)
   767→}
   768→
   769→// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
   770→func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
   771→	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
   772→		return err
   773→	}
   774→	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
   775→		return err
   776→	}
   777→	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
   778→		return err
   779→	}
   780→	return writeBinaryLevel(w, cf)
   781→}
   782→
   783→// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
   784→func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
   785→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
   786→		return err
   787→	}
   788→
   789→	for name, metric := range cf.Metrics {
   790→		if err := writeString16(w, name); err != nil {
   791→			return err
   792→		}
   793→		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
   794→			return err
   795→		}
   796→		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
   797→			return err
   798→		}
   799→		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
   800→			return err
   801→		}
   802→		for _, v := range metric.Data {
   803→			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
   804→				return err
   805→			}
   806→		}
   807→	}
   808→
   809→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
   810→		return err
   811→	}
   812→
   813→	for name, child := range cf.Children {
   814→		if err := writeString16(w, name); err != nil {
   815→			return err
   816→		}
   817→		if err := writeBinaryLevel(w, child); err != nil {
   818→			return err
   819→		}
   820→	}
   821→
   822→	return nil
   823→}
   824→
   825→// writeString16 writes a 2-byte length-prefixed string to w.
   826→func writeString16(w io.Writer, s string) error {
   827→	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
   828→		return err
   829→	}
   830→	_, err := io.WriteString(w, s)
   831→	return err
   832→}
   833→
   834→// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
   835→// The retention check (from) is applied to the file's 'to' timestamp.
   836→func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
   837→	br := bufio.NewReader(f)
   838→
   839→	var magic uint32
   840→	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
   841→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
   842→	}
   843→	if magic != snapFileMagic {
   844→		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
   845→	}
   846→
   847→	var fileFrom, fileTo int64
   848→	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
   849→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
   850→	}
   851→	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
   852→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
   853→	}
   854→
   855→	if fileTo != 0 && fileTo < from {
   856→		return nil // File is older than retention window, skip it
   857→	}
   858→
   859→	cf, err := readBinaryLevel(br)
   860→	if err != nil {
   861→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
   862→	}
   863→	cf.From = fileFrom
   864→	cf.To = fileTo
   865→
   866→	return l.loadFile(cf, m)
   867→}
   868→
   869→// readBinaryLevel recursively reads a level from the binary snapshot format.
   870→func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
   871→	cf := &CheckpointFile{
   872→		Metrics:  make(map[string]*CheckpointMetrics),
   873→		Children: make(map[string]*CheckpointFile),
   874→	}
   875→
   876→	var numMetrics uint32
   877→	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
   878→		return nil, fmt.Errorf("read num_metrics: %w", err)
   879→	}
   880→
   881→	for range numMetrics {
   882→		name, err := readString16(r)
   883→		if err != nil {
   884→			return nil, fmt.Errorf("read metric name: %w", err)
   885→		}
   886→
   887→		var freq, start int64
   888→		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
   889→			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
   890→		}
   891→		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
   892→			return nil, fmt.Errorf("read start for %s: %w", name, err)
   893→		}
   894→
   895→		var numValues uint32
   896→		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
   897→			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
   898→		}
   899→
   900→		data := make([]schema.Float, numValues)
   901→		for i := range numValues {
   902→			var bits uint32
   903→			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
   904→				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
   905→			}
   906→			data[i] = schema.Float(math.Float32frombits(bits))
   907→		}
   908→
   909→		cf.Metrics[name] = &CheckpointMetrics{
   910→			Frequency: freq,
   911→			Start:     start,
   912→			Data:      data,
   913→		}
   914→	}
   915→
   916→	var numChildren uint32
   917→	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
   918→		return nil, fmt.Errorf("read num_children: %w", err)
   919→	}
   920→
   921→	for range numChildren {
   922→		childName, err := readString16(r)
   923→		if err != nil {
   924→			return nil, fmt.Errorf("read child name: %w", err)
   925→		}
   926→
   927→		child, err := readBinaryLevel(r)
   928→		if err != nil {
   929→			return nil, fmt.Errorf("read child %s: %w", childName, err)
   930→		}
   931→		cf.Children[childName] = child
   932→	}
   933→
   934→	return cf, nil
   935→}
   936→
   937→// readString16 reads a 2-byte length-prefixed string from r.
   938→func readString16(r io.Reader) (string, error) {
   939→	var sLen uint16
   940→	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
   941→		return "", err
   942→	}
   943→	buf := make([]byte, sLen)
   944→	if _, err := io.ReadFull(r, buf); err != nil {
   945→		return "", err
   946→	}
   947→	return string(buf), nil
   948→}
   949→