     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides an efficient in-memory time-series metric storage system
     7→// with support for hierarchical data organization, checkpointing, and archiving.
     8→//
     9→// The package organizes metrics in a tree structure (cluster → host → component) and
    10→// provides concurrent read/write access to metric data with configurable aggregation strategies.
    11→// Background goroutines handle periodic checkpointing (JSON or Avro format), archiving old data,
    12→// and enforcing retention policies.
    13→//
    14→// Key features:
    15→//   - In-memory metric storage with configurable retention
    16→//   - Hierarchical data organization (selectors)
    17→//   - Concurrent checkpoint/archive workers
    18→//   - Support for sum and average aggregation
    19→//   - NATS integration for metric ingestion
    20→package metricstore
    21→
    22→import (
    23→	"bytes"
    24→	"context"
    25→	"encoding/json"
    26→	"errors"
    27→	"fmt"
    28→	"runtime"
    29→	"runtime/debug"
    30→	"slices"
    31→	"sync"
    32→	"time"
    33→
    34→	"github.com/ClusterCockpit/cc-backend/internal/config"
    35→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    36→	"github.com/ClusterCockpit/cc-lib/v2/resampler"
    37→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    38→	"github.com/ClusterCockpit/cc-lib/v2/util"
    39→)
    40→
    41→// GlobalState holds the global state for the metric store with thread-safe access.
    42→type GlobalState struct {
    43→	mu                sync.RWMutex
    44→	lastRetentionTime int64
    45→	selectorsExcluded bool
    46→}
    47→
    48→var (
    49→	singleton  sync.Once
    50→	msInstance *MemoryStore
    51→	// shutdownFunc stores the context cancellation function created in Init
    52→	// and is called during Shutdown to cancel all background goroutines
    53→	shutdownFunc   context.CancelFunc
    54→	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
    55→	// Create a global instance
    56→	state = &GlobalState{}
    57→)
    58→
    59→// NodeProvider provides information about nodes currently in use by running jobs.
    60→//
    61→// This interface allows metricstore to query job information without directly
    62→// depending on the repository package, breaking the import cycle.
    63→//
    64→// Implementations should return nodes that are actively processing jobs started
    65→// before the given timestamp. These nodes will be excluded from retention-based
    66→// garbage collection to prevent data loss for jobs that are still running or
    67→// recently completed.
    68→type NodeProvider interface {
    69→	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
    70→	// that are currently in use by jobs that started before the given timestamp.
    71→	//
    72→	// Parameters:
    73→	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
    74→	//
    75→	// Returns:
    76→	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
    77→	//   - Error if the query fails
    78→	GetUsedNodes(ts int64) (map[string][]string, error)
    79→}
    80→
    81→// Metric represents a single metric data point to be written to the store.
    82→type Metric struct {
    83→	Name  string
    84→	Value schema.Float
    85→	// MetricConfig contains frequency and aggregation settings for this metric.
    86→	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
    87→	MetricConfig MetricConfig
    88→}
    89→
    90→// MemoryStore is the main in-memory time-series metric storage implementation.
    91→//
    92→// It organizes metrics in a hierarchical tree structure where each level represents
    93→// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
    94→// store multiple metrics as time-series buffers.
    95→//
    96→// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
    97→// All public methods are safe for concurrent use.
    98→type MemoryStore struct {
    99→	Metrics      map[string]MetricConfig
   100→	root         Level
   101→	nodeProvider NodeProvider
   102→}
   103→
   104→// Init initializes the metric store from configuration and starts background workers.
   105→//
   106→// This function must be called exactly once before any other metricstore operations.
   107→// It performs the following initialization steps:
   108→//  1. Validates and decodes the metric store configuration
   109→//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
   110→//  3. Loads metric configurations from all registered clusters
   111→//  4. Restores checkpoints within the retention window
   112→//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
   113→//  6. Optionally subscribes to NATS for real-time metric ingestion
   114→//
   115→// Parameters:
   116→//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig)
   117→//   - wg: WaitGroup that will be incremented for each background goroutine started
   118→//
   119→// The function will call cclog.Fatal on critical errors during initialization.
   120→// Use Shutdown() to cleanly stop all background workers started by Init().
   121→//
   122→// Note: Signal handling must be implemented by the caller. Call Shutdown() when
   123→// receiving termination signals to ensure checkpoint data is persisted.
   124→func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
   125→	startupTime := time.Now()
   126→
   127→	if rawConfig != nil {
   128→		config.Validate(configSchema, rawConfig)
   129→		dec := json.NewDecoder(bytes.NewReader(rawConfig))
   130→		dec.DisallowUnknownFields()
   131→		if err := dec.Decode(&Keys); err != nil {
   132→			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
   133→		}
   134→	}
   135→
   136→	// Set NumWorkers from config or use default
   137→	if Keys.NumWorkers <= 0 {
   138→		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
   139→	}
   140→	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
   141→
   142→	// Pass the config.MetricStoreKeys
   143→	InitMetrics(metrics)
   144→
   145→	ms := GetMemoryStore()
   146→
   147→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   148→	if err != nil {
   149→		cclog.Fatal(err)
   150→	}
   151→
   152→	restoreFrom := startupTime.Add(-d)
   153→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   154→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   155→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   156→	if err != nil {
   157→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   158→	} else {
   159→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   160→	}
   161→
   162→	// Try to use less memory by forcing a GC run here and then
   163→	// lowering the target percentage. The default of 100 means
   164→	// that only once the ratio of new allocations execeds the
   165→	// previously active heap, a GC is triggered.
   166→	// Forcing a GC here will set the "previously active heap"
   167→	// to a minumum.
   168→	// runtime.GC()
   169→
   170→	ctx, shutdown := context.WithCancel(context.Background())
   171→
   172→	Retention(wg, ctx)
   173→	Checkpointing(wg, ctx)
   174→	CleanUp(wg, ctx)
   175→	WALStaging(wg, ctx)
   176→	MemoryUsageTracker(wg, ctx)
   177→
   178→	// Note: Signal handling has been removed from this function.
   179→	// The caller is responsible for handling shutdown signals and calling
   180→	// the shutdown() function when appropriate.
   181→	// Store the shutdown function for later use by Shutdown()
   182→	shutdownFuncMu.Lock()
   183→	shutdownFunc = shutdown
   184→	shutdownFuncMu.Unlock()
   185→
   186→	if Keys.Subscriptions != nil {
   187→		wg.Go(func() {
   188→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   189→				cclog.Fatal(err)
   190→			}
   191→		})
   192→	}
   193→}
   194→
   195→// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
   196→//
   197→// This function must be called before GetMemoryStore() and can only be called once due to
   198→// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
   199→//
   200→// Parameters:
   201→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
   202→//
   203→// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
   204→//
   205→// After this call, the global msInstance is ready for use via GetMemoryStore().
   206→func InitMetrics(metrics map[string]MetricConfig) {
   207→	singleton.Do(func() {
   208→		offset := 0
   209→		for key, cfg := range metrics {
   210→			if cfg.Frequency == 0 {
   211→				panic("[METRICSTORE]> invalid frequency")
   212→			}
   213→
   214→			metrics[key] = MetricConfig{
   215→				Frequency:   cfg.Frequency,
   216→				Aggregation: cfg.Aggregation,
   217→				offset:      offset,
   218→			}
   219→			offset += 1
   220→		}
   221→
   222→		msInstance = &MemoryStore{
   223→			root: Level{
   224→				metrics:  make([]*buffer, len(metrics)),
   225→				children: make(map[string]*Level),
   226→			},
   227→			Metrics: metrics,
   228→		}
   229→	})
   230→}
   231→
   232→// GetMemoryStore returns the singleton MemoryStore instance.
   233→//
   234→// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
   235→//
   236→// This function is safe for concurrent use after initialization.
   237→func GetMemoryStore() *MemoryStore {
   238→	if msInstance == nil {
   239→		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
   240→	}
   241→
   242→	return msInstance
   243→}
   244→
   245→func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
   246→	if metric, ok := ms.Metrics[metricName]; ok {
   247→		return metric.Frequency, nil
   248→	}
   249→	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
   250→}
   251→
   252→// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
   253→// This must be called during initialization to provide job state information
   254→// for selective buffer retention during Free operations.
   255→// If not set, the Free function will fall back to freeing all buffers.
   256→func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
   257→	ms.nodeProvider = provider
   258→}
   259→
   260→// Shutdown performs a graceful shutdown of the metric store.
   261→//
   262→// This function cancels all background goroutines started by Init() and writes
   263→// a final checkpoint to disk before returning. It should be called when the
   264→// application receives a termination signal.
   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→
   290→	ms := GetMemoryStore()
   291→
   292→	lastCheckpointMu.Lock()
   293→	from := lastCheckpoint
   294→	lastCheckpointMu.Unlock()
   295→
   296→	if Keys.Checkpoints.FileFormat == "wal" {
   297→		var hostDirs []string
   298→		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   299→		if err == nil {
   300→			RotateWALFilesAfterShutdown(hostDirs)
   301→		}
   302→	} else {
   303→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   304→	}
   305→
   306→	if err != nil {
   307→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
   308→	}
   309→	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
   310→}
   311→
   312→// Retention starts a background goroutine that periodically frees old metric data.
   313→//
   314→// This worker runs at half the retention interval and calls Free() to remove buffers
   315→// older than the configured retention time. It respects the NodeProvider to preserve
   316→// data for nodes with active jobs.
   317→//
   318→// Parameters:
   319→//   - wg: WaitGroup to signal completion when context is cancelled
   320→//   - ctx: Context for cancellation signal
   321→//
   322→// The goroutine exits when ctx is cancelled.
   323→func Retention(wg *sync.WaitGroup, ctx context.Context) {
   324→	ms := GetMemoryStore()
   325→
   326→	wg.Go(func() {
   327→		d, err := time.ParseDuration(Keys.RetentionInMemory)
   328→		if err != nil {
   329→			cclog.Fatal(err)
   330→		}
   331→		if d <= 0 {
   332→			return
   333→		}
   334→
   335→		tickInterval := d / 2
   336→		if tickInterval <= 0 {
   337→			return
   338→		}
   339→		ticker := time.NewTicker(tickInterval)
   340→		defer ticker.Stop()
   341→
   342→		for {
   343→			select {
   344→			case <-ctx.Done():
   345→				return
   346→			case <-ticker.C:
   347→				state.mu.Lock()
   348→
   349→				t := time.Now().Add(-d)
   350→
   351→				state.lastRetentionTime = t.Unix()
   352→
   353→				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
   354→
   355→				freed, err := Free(ms, t)
   356→				if err != nil {
   357→					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
   358→				} else {
   359→					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
   360→				}
   361→
   362→				state.mu.Unlock()
   363→
   364→				// Clean up the buffer pool
   365→				bufferPool.Clean(state.lastRetentionTime)
   366→			}
   367→		}
   368→	})
   369→}
   370→
   371→// MemoryUsageTracker starts a background goroutine that monitors memory usage.
   372→//
   373→// This worker checks actual process memory usage (via runtime.MemStats) periodically
   374→// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
   375→// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
   376→// performance issues.
   377→//
   378→// The tracker logs both actual memory usage (heap allocated) and metric data size for
   379→// visibility into memory overhead from Go runtime structures and allocations.
   380→//
   381→// Parameters:
   382→//   - wg: WaitGroup to signal completion when context is cancelled
   383→//   - ctx: Context for cancellation signal
   384→//
   385→// The goroutine exits when ctx is cancelled.
   386→func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
   387→	ms := GetMemoryStore()
   388→
   389→	wg.Go(func() {
   390→		normalInterval := DefaultMemoryUsageTrackerInterval
   391→		fastInterval := 30 * time.Second
   392→
   393→		if normalInterval <= 0 {
   394→			return
   395→		}
   396→
   397→		ticker := time.NewTicker(normalInterval)
   398→		defer ticker.Stop()
   399→		currentInterval := normalInterval
   400→
   401→		for {
   402→			select {
   403→			case <-ctx.Done():
   404→				return
   405→			case <-ticker.C:
   406→				var mem runtime.MemStats
   407→				runtime.ReadMemStats(&mem)
   408→				actualMemoryGB := float64(mem.Alloc) / 1e9
   409→				metricDataGB := ms.SizeInGB()
   410→				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
   411→
   412→				freedExcluded := 0
   413→				freedEmergency := 0
   414→				var err error
   415→
   416→				state.mu.RLock()
   417→				lastRetention := state.lastRetentionTime
   418→				selectorsExcluded := state.selectorsExcluded
   419→				state.mu.RUnlock()
   420→
   421→				if lastRetention != 0 && selectorsExcluded {
   422→					freedExcluded, err = ms.Free(nil, lastRetention)
   423→					if err != nil {
   424→						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
   425→					}
   426→
   427→					if freedExcluded > 0 {
   428→						debug.FreeOSMemory()
   429→						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
   430→					}
   431→				}
   432→
   433→				runtime.ReadMemStats(&mem)
   434→				actualMemoryGB = float64(mem.Alloc) / 1e9
   435→
   436→				if actualMemoryGB > float64(Keys.MemoryCap) {
   437→					cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)
   438→
   439→					// Use progressive time-based Free with increasing threshold
   440→					// instead of ForceFree loop — fewer tree traversals, more effective
   441→					d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
   442→					if parseErr != nil {
   443→						cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
   444→					} else {
   445→						thresholds := []float64{0.75, 0.5, 0.25}
   446→						for _, fraction := range thresholds {
   447→							threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
   448→							freed, freeErr := ms.Free(nil, threshold.Unix())
   449→							if freeErr != nil {
   450→								cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
   451→							}
   452→							freedEmergency += freed
   453→
   454→							bufferPool.Clear()
   455→							runtime.GC()
   456→							runtime.ReadMemStats(&mem)
   457→							actualMemoryGB = float64(mem.Alloc) / 1e9
   458→
   459→							if actualMemoryGB < float64(Keys.MemoryCap) {
   460→								break
   461→							}
   462→						}
   463→					}
   464→
   465→					bufferPool.Clear()
   466→					debug.FreeOSMemory()
   467→
   468→					runtime.ReadMemStats(&mem)
   469→					actualMemoryGB = float64(mem.Alloc) / 1e9
   470→
   471→					if actualMemoryGB >= float64(Keys.MemoryCap) {
   472→						cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
   473→					} else {
   474→						cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
   475→					}
   476→				}
   477→
   478→				// Adaptive ticker: check more frequently when memory is high
   479→				memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
   480→				if memoryRatio > 0.8 && currentInterval != fastInterval {
   481→					ticker.Reset(fastInterval)
   482→					currentInterval = fastInterval
   483→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
   484→				} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
   485→					ticker.Reset(normalInterval)
   486→					currentInterval = normalInterval
   487→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
   488→				}
   489→			}
   490→		}
   491→	})
   492→}
   493→
   494→// Free removes metric data older than the given time while preserving data for active nodes.
   495→//
   496→// This function implements intelligent retention by consulting the NodeProvider (if configured)
   497→// to determine which nodes are currently in use by running jobs. Data for these nodes is
   498→// preserved even if older than the retention time.
   499→//
   500→// Parameters:
   501→//   - ms: The MemoryStore instance
   502→//   - t: Time threshold - buffers with data older than this will be freed
   503→//
   504→// Returns:
   505→//   - Number of buffers freed
   506→//   - Error if NodeProvider query fails
   507→//
   508→// Behavior:
   509→//   - If no NodeProvider is set: frees all buffers older than t
   510→//   - If NodeProvider returns empty map: frees all buffers older than t
   511→//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
   512→func Free(ms *MemoryStore, t time.Time) (int, error) {
   513→	// If no NodeProvider is configured, free all buffers older than t
   514→	if ms.nodeProvider == nil {
   515→		return ms.Free(nil, t.Unix())
   516→	}
   517→
   518→	excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
   519→	if err != nil {
   520→		return 0, err
   521→	}
   522→
   523→	switch lenMap := len(excludeSelectors); lenMap {
   524→
   525→	// If the length of the map returned by GetUsedNodes() is 0,
   526→	// then use default Free method with nil selector
   527→	case 0:
   528→		state.selectorsExcluded = false
   529→		return ms.Free(nil, t.Unix())
   530→
   531→	// Else formulate selectors, exclude those from the map
   532→	// and free the rest of the selectors
   533→	default:
   534→		state.selectorsExcluded = true
   535→		selectors := GetSelectors(ms, excludeSelectors)
   536→		return FreeSelected(ms, selectors, t)
   537→	}
   538→}
   539→
   540→// FreeSelected frees buffers for specific selectors while preserving others.
   541→//
   542→// This function is used when we want to retain some specific nodes beyond the retention time.
   543→// It iterates through the provided selectors and frees their associated buffers.
   544→//
   545→// Parameters:
   546→//   - ms: The MemoryStore instance
   547→//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
   548→//   - t: Time threshold for freeing buffers
   549→//
   550→// Returns the total number of buffers freed and any error encountered.
   551→func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
   552→	freed := 0
   553→
   554→	for _, selector := range selectors {
   555→
   556→		freedBuffers, err := ms.Free(selector, t.Unix())
   557→		if err != nil {
   558→			cclog.Errorf("error while freeing selected buffers: %#v", err)
   559→		}
   560→		freed += freedBuffers
   561→
   562→	}
   563→
   564→	return freed, nil
   565→}
   566→
   567→// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
   568→//
   569→// This function generates a list of selectors whose buffers should be freed by excluding
   570→// selectors that correspond to nodes currently in use by running jobs.
   571→//
   572→// Parameters:
   573→//   - ms: The MemoryStore instance
   574→//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
   575→//
   576→// Returns a list of selectors ([]string paths) that can be safely freed.
   577→//
   578→// Example:
   579→//
   580→//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
   581→//	and excludeSelectors contains {"emmy": ["node001"]},
   582→//	then only [["emmy", "node002"]] is returned.
   583→func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
   584→	allSelectors := ms.GetPaths(2)
   585→
   586→	filteredSelectors := make([][]string, 0, len(allSelectors))
   587→
   588→	for _, path := range allSelectors {
   589→		if len(path) < 2 {
   590→			continue
   591→		}
   592→
   593→		key := path[0]   // The "Key" (Level 1)
   594→		value := path[1] // The "Value" (Level 2)
   595→
   596→		exclude := false
   597→
   598→		// Check if the key exists in our exclusion map
   599→		if excludedValues, exists := excludeSelectors[key]; exists {
   600→			// The key exists, now check if the specific value is in the exclusion list
   601→			if slices.Contains(excludedValues, value) {
   602→				exclude = true
   603→			}
   604→		}
   605→
   606→		if !exclude {
   607→			filteredSelectors = append(filteredSelectors, path)
   608→		}
   609→	}
   610→
   611→	return filteredSelectors
   612→}
   613→
   614→// GetPaths returns a list of lists (paths) to the specified depth.
   615→func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
   616→	var results [][]string
   617→
   618→	// Start recursion. Initial path is empty.
   619→	// We treat Root as depth 0.
   620→	ms.root.collectPaths(0, targetDepth, []string{}, &results)
   621→
   622→	return results
   623→}
   624→
   625→// Write all values in `metrics` to the level specified by `selector` for time `ts`.
   626→// Look at `findLevelOrCreate` for how selectors work.
   627→func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
   628→	var ok bool
   629→	for i, metric := range metrics {
   630→		if metric.MetricConfig.Frequency == 0 {
   631→			metric.MetricConfig, ok = m.Metrics[metric.Name]
   632→			if !ok {
   633→				cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
   634→				metric.MetricConfig.Frequency = 0
   635→			}
   636→			metrics[i] = metric
   637→		}
   638→	}
   639→
   640→	return m.WriteToLevel(&m.root, selector, ts, metrics)
   641→}
   642→
   643→func (m *MemoryStore) GetLevel(selector []string) *Level {
   644→	return m.root.findLevelOrCreate(selector, len(m.Metrics))
   645→}
   646→
   647→// WriteToLevel assumes that `minfo` in `metrics` is filled in
   648→func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
   649→	l = l.findLevelOrCreate(selector, len(m.Metrics))
   650→	l.lock.Lock()
   651→	defer l.lock.Unlock()
   652→
   653→	for _, metric := range metrics {
   654→		if metric.MetricConfig.Frequency == 0 {
   655→			continue
   656→		}
   657→
   658→		b := l.metrics[metric.MetricConfig.offset]
   659→		if b == nil {
   660→			// First write to this metric and level
   661→			b = newBuffer(ts, metric.MetricConfig.Frequency)
   662→			l.metrics[metric.MetricConfig.offset] = b
   663→		}
   664→
   665→		nb, err := b.write(ts, metric.Value)
   666→		if err != nil {
   667→			return err
   668→		}
   669→
   670→		// Last write created a new buffer...
   671→		if b != nb {
   672→			l.metrics[metric.MetricConfig.offset] = nb
   673→		}
   674→	}
   675→	return nil
   676→}
   677→
   678→// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
   679→// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
   680→// The second and third return value are the actual from/to for the data. Those can be different from
   681→// the range asked for if no data was available.
   682→func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
   683→	if from > to {
   684→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
   685→	}
   686→
   687→	minfo, ok := m.Metrics[metric]
   688→	if !ok {
   689→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
   690→	}
   691→
   692→	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
   693→
   694→	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
   695→		cdata, cfrom, cto, err := b.read(from, to, data)
   696→		if err != nil {
   697→			return err
   698→		}
   699→
   700→		if n == 0 {
   701→			from, to = cfrom, cto
   702→		} else if from != cfrom || to != cto || len(data) != len(cdata) {
   703→			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
   704→			if missingfront != 0 {
   705→				return ErrDataDoesNotAlign
   706→			}
   707→
   708→			newlen := len(cdata) - missingback
   709→			if newlen < 1 {
   710→				return ErrDataDoesNotAlign
   711→			}
   712→			cdata = cdata[0:newlen]
   713→			if len(cdata) != len(data) {
   714→				return ErrDataDoesNotAlign
   715→			}
   716→
   717→			from, to = cfrom, cto
   718→		}
   719→
   720→		data = cdata
   721→		n += 1
   722→		return nil
   723→	})
   724→
   725→	if err != nil {
   726→		return nil, 0, 0, 0, err
   727→	} else if n == 0 {
   728→		return nil, 0, 0, 0, ErrNoHostOrMetric
   729→	} else if n > 1 {
   730→		if minfo.Aggregation == AvgAggregation {
   731→			normalize := 1. / schema.Float(n)
   732→			for i := 0; i < len(data); i++ {
   733→				data[i] *= normalize
   734→			}
   735→		} else if minfo.Aggregation != SumAggregation {
   736→			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
   737→		}
   738→	}
   739→
   740→	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
   741→	if err != nil {
   742→		return nil, 0, 0, 0, err
   743→	}
   744→
   745→	return data, from, to, resolution, nil
   746→}
   747→
   748→// Free releases all buffers for the selected level and all its children that
   749→// contain only values older than `t`.
   750→func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
   751→	return m.GetLevel(selector).free(t)
   752→}
   753→
   754→// ForceFree unconditionally removes the oldest buffer from each metric chain.
   755→func (m *MemoryStore) ForceFree() (int, error) {
   756→	return m.GetLevel(nil).forceFree()
   757→}
   758→
   759→func (m *MemoryStore) FreeAll() error {
   760→	m.root.lock.Lock()
   761→	defer m.root.lock.Unlock()
   762→
   763→	for k := range m.root.children {
   764→		delete(m.root.children, k)
   765→	}
   766→
   767→	return nil
   768→}
   769→
   770→func (m *MemoryStore) SizeInBytes() int64 {
   771→	return m.root.sizeInBytes()
   772→}
   773→
   774→func (m *MemoryStore) SizeInGB() float64 {
   775→	return float64(m.root.sizeInBytes()) / 1e9
   776→}
   777→
   778→// ListChildren , given a selector, returns a list of all children of the level
   779→// selected.
   780→func (m *MemoryStore) ListChildren(selector []string) []string {
   781→	lvl := &m.root
   782→	for lvl != nil && len(selector) != 0 {
   783→		lvl.lock.RLock()
   784→		next := lvl.children[selector[0]]
   785→		lvl.lock.RUnlock()
   786→		lvl = next
   787→		selector = selector[1:]
   788→	}
   789→
   790→	if lvl == nil {
   791→		return nil
   792→	}
   793→
   794→	lvl.lock.RLock()
   795→	defer lvl.lock.RUnlock()
   796→
   797→	children := make([]string, 0, len(lvl.children))
   798→	for child := range lvl.children {
   799→		children = append(children, child)
   800→	}
   801→
   802→	return children
   803→}
   804→


Bash

git log --oneline --all -- pkg/metricstore/walCheckpoint.go | head -20


6f7dda53 Cleanup
0325d9e8 fix: Increase throughput for WAL writers
96a79098 carry forward: uncommitted session files
d3b94c33 Completed 'Explore' agent: Find CUDA/HIP macro switching (toolu_01EbN3)
bf1a8a17 fix: Shard WAL consumer for higher throughput
50aed595 fix: metricstore NATS contention
818a05f2 carry forward: uncommitted session files
b214e175 Add buffered I/O to WAL writes and fix MemoryCap comment
ecd29b28 carry forward: uncommitted session files
21b6c1b5 carry forward: uncommitted session files
a243e174 Update to shutdown worker for WAL checkpointing mode
a418abc7 Run go fix
ca0f9a42 Introduce metric store binary checkpoints with write ahead log


commit bf1a8a174eca262d3f3d1179df3bff86f276d9a3
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Wed Mar 18 06:32:14 2026 +0100

    fix: Shard WAL consumer for higher throughput

    Entire-Checkpoint: e583b7b11439

 pkg/metricstore/lineprotocol.go  |   8 +-
 pkg/metricstore/metricstore.go   |   4 +-
 pkg/metricstore/walCheckpoint.go | 362 +++++++++++++++++++++++++++------------
 3 files changed, 254 insertions(+), 120 deletions(-)


git show bf1a8a17:pkg/metricstore/walCheckpoint.go | head -200


// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼
//	        WALStaging goroutine
//	               │
//	               ▼
//	        checkpoints/cluster/host/current.wal  (append-only, binary)
//
//	Periodic checkpoint (Checkpointing goroutine):
//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
//	  2. Signal WALStaging to truncate current.wal per host
//
//	On restart (FromCheckpoint):
//	  1. Load most recent <timestamp>.bin snapshot
//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
//
// # WAL Record Format
//
//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
//
//	payload:
//	  [8B timestamp int64]
//	  [2B metric_name_len uint16][N metric name bytes]
//	  [1B selector_count uint8]
//	  per selector: [1B selector_len uint8][M selector bytes]
//	  [4B value float32 bits]
//
// # Binary Snapshot Format
//
//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
//	Level tree (recursive):
//	  [4B num_metrics uint32]
//	  per metric:
//	    [2B name_len uint16][N name bytes]
//	    [8B frequency int64][8B start int64]
//	    [4B num_values uint32][num_values × 4B float32]
//	  [4B num_children uint32]
//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
package metricstore

import (
	"bufio"
	"context"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"hash/fnv"
	"io"
	"math"
	"os"
	"path"
	"strings"
	"sync"
	"sync/atomic"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Magic numbers for binary formats.
const (
	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
)

// walShardChs holds per-shard channels for WAL messages.
// Initialized by WALStaging; nil when WAL is not active.
var walShardChs []chan *WALMessage

// walShardRotateChs holds per-shard channels for WAL rotation requests.
var walShardRotateChs []chan walRotateReq

// walNumShards stores the number of shards (set during WALStaging init).
var walNumShards int

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f *os.File
	w *bufio.Writer
}

// walShardIndex computes which shard a message belongs to based on cluster+node.
// Uses FNV-1a hash for fast, well-distributed mapping.
func walShardIndex(cluster, node string) int {
	h := fnv.New32a()
	h.Write([]byte(cluster))
	h.Write([]byte{0})
	h.Write([]byte(node))
	return int(h.Sum32()) % walNumShards
}

// SendWALMessage routes a WAL message to the appropriate shard channel.
// Returns false if the channel is full (message dropped).
func SendWALMessage(msg *WALMessage) bool {
	if walShardChs == nil {
		return false
	}
	shard := walShardIndex(msg.Cluster, msg.Node)
	select {
	case walShardChs[shard] <- msg:
		return true
	default:
		return false
	}
}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.
// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return
	}

	walNumShards = max(Keys.NumWorkers, 1)
	chBufSize := max(65536/walNumShards, 1024)

	walShardChs = make([]chan *WALMessage, walNumShards)
	walShardRotateChs = make([]chan walRotateReq, walNumShards)

	for i := range walNumShards {
		walShardChs[i] = make(chan *WALMessage, chBufSize)
		walShardRotateChs[i] = make(chan walRotateReq, 64)
	}

	for i := range walNumShards {
		msgCh := walShardChs[i]
		rotateCh := walShardRotateChs[i]

		wg.Go(func() {
			hostFiles := make(map[string]*walFileState)

			defer func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
						ws.f.Close()
					}
				}
			}()

			getOrOpenWAL := func(hostDir string) *walFileState {
				ws, ok := hostFiles[hostDir]
				if ok {
					return ws
				}

				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
					return nil
				}

				walPath := path.Join(hostDir, "current.wal")
				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
					return nil
				}

				w := bufio.NewWriter(f)

				// Write file header magic if file is new (empty).
				info, err := f.Stat()


Found 1 file
pkg/metricstore/checkpoint.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements checkpoint persistence for the in-memory metric store.
     7→//
     8→// Checkpoints enable graceful restarts by periodically saving in-memory metric
     9→// data to disk in JSON or binary format. The checkpoint system:
    10→//
    11→// Key Features:
    12→//   - Periodic background checkpointing via the Checkpointing() worker
    13→//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
    14→//   - Parallel checkpoint creation and loading using worker pools
    15→//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
    16→//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
    17→//   - Only saves unarchived data (archived data is already persisted elsewhere)
    18→//   - GC optimization during loading to prevent excessive heap growth
    19→//
    20→// Checkpoint Workflow:
    21→//  1. Init() loads checkpoints within retention window at startup
    22→//  2. Checkpointing() worker periodically saves new data
    23→//  3. Shutdown() writes final checkpoint before exit
    24→//
    25→// File Organization:
    26→//
    27→//	checkpoints/
    28→//	  cluster1/
    29→//	    host001/
    30→//	      1234567890.json  (JSON format: full subtree snapshot)
    31→//	      1234567890.bin   (binary format: full subtree snapshot)
    32→//	      current.wal      (WAL format: append-only per-entry log)
    33→//	    host002/
    34→//	      ...
    35→package metricstore
    36→
    37→import (
    38→	"bufio"
    39→	"context"
    40→	"encoding/json"
    41→	"errors"
    42→	"fmt"
    43→	"io/fs"
    44→	"os"
    45→	"path"
    46→	"path/filepath"
    47→	"sort"
    48→	"strconv"
    49→	"strings"
    50→	"sync"
    51→	"sync/atomic"
    52→	"time"
    53→
    54→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    55→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    56→)
    57→
    58→const (
    59→	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
    60→	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
    61→	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
    62→)
    63→
    64→// CheckpointMetrics represents metric data in a checkpoint file.
    65→// Whenever the structure changes, update MarshalJSON as well!
    66→type CheckpointMetrics struct {
    67→	Data      []schema.Float `json:"data"`
    68→	Frequency int64          `json:"frequency"`
    69→	Start     int64          `json:"start"`
    70→}
    71→
    72→// CheckpointFile represents the hierarchical structure of a checkpoint file.
    73→// It mirrors the Level tree structure from the MemoryStore.
    74→type CheckpointFile struct {
    75→	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
    76→	Children map[string]*CheckpointFile    `json:"children"`
    77→	From     int64                         `json:"from"`
    78→	To       int64                         `json:"to"`
    79→}
    80→
    81→// lastCheckpoint tracks the timestamp of the last checkpoint creation.
    82→var (
    83→	lastCheckpoint   time.Time
    84→	lastCheckpointMu sync.Mutex
    85→)
    86→
    87→// Checkpointing starts a background worker that periodically saves metric data to disk.
    88→//
    89→// Checkpoints are written every 12 hours (hardcoded).
    90→//
    91→// Format behaviour:
    92→//   - "json": Periodic checkpointing every checkpointInterval
    93→//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
    94→func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
    95→	lastCheckpointMu.Lock()
    96→	lastCheckpoint = time.Now()
    97→	lastCheckpointMu.Unlock()
    98→
    99→	ms := GetMemoryStore()
   100→
   101→	wg.Go(func() {
   102→
   103→		d := 12 * time.Hour // default checkpoint interval
   104→		if Keys.CheckpointInterval != "" {
   105→			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
   106→			if err != nil {
   107→				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
   108→			} else {
   109→				d = parsed
   110→			}
   111→		}
   112→
   113→		ticker := time.NewTicker(d)
   114→		defer ticker.Stop()
   115→
   116→		for {
   117→			select {
   118→			case <-ctx.Done():
   119→				return
   120→			case <-ticker.C:
   121→				lastCheckpointMu.Lock()
   122→				from := lastCheckpoint
   123→				lastCheckpointMu.Unlock()
   124→
   125→				now := time.Now()
   126→				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
   127→
   128→				if Keys.Checkpoints.FileFormat == "wal" {
   129→					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   130→					if err != nil {
   131→						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
   132→					} else {
   133→						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
   134→						lastCheckpointMu.Lock()
   135→						lastCheckpoint = now
   136→						lastCheckpointMu.Unlock()
   137→						// Rotate WAL files for successfully checkpointed hosts.
   138→						RotateWALFiles(hostDirs)
   139→					}
   140→				} else {
   141→					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   142→					if err != nil {
   143→						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
   144→					} else {
   145→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
   146→						lastCheckpointMu.Lock()
   147→						lastCheckpoint = now
   148→						lastCheckpointMu.Unlock()
   149→					}
   150→				}
   151→			}
   152→		}
   153→	})
   154→}
   155→
   156→// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
   157→//
   158→// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
   159→// This method manually constructs JSON to avoid allocations and interface conversions.
   160→func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
   161→	buf := make([]byte, 0, 128+len(cm.Data)*8)
   162→	buf = append(buf, `{"frequency":`...)
   163→	buf = strconv.AppendInt(buf, cm.Frequency, 10)
   164→	buf = append(buf, `,"start":`...)
   165→	buf = strconv.AppendInt(buf, cm.Start, 10)
   166→	buf = append(buf, `,"data":[`...)
   167→	for i, x := range cm.Data {
   168→		if i != 0 {
   169→			buf = append(buf, ',')
   170→		}
   171→		if x.IsNaN() {
   172→			buf = append(buf, `null`...)
   173→		} else {
   174→			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
   175→		}
   176→	}
   177→	buf = append(buf, `]}`...)
   178→	return buf, nil
   179→}
   180→
   181→// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format).
   182→//
   183→// Metrics at root and cluster levels are skipped. One file per host is created.
   184→// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
   185→// at a time, allowing concurrent writes/reads to other hosts.
   186→//
   187→// Returns the number of checkpoint files created and any errors encountered.
   188→func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
   189→	// Pre-calculate capacity by counting cluster/host pairs
   190→	m.root.lock.RLock()
   191→	totalHosts := 0
   192→	for _, l1 := range m.root.children {
   193→		l1.lock.RLock()
   194→		totalHosts += len(l1.children)
   195→		l1.lock.RUnlock()
   196→	}
   197→	m.root.lock.RUnlock()
   198→
   199→	levels := make([]*Level, 0, totalHosts)
   200→	selectors := make([][]string, 0, totalHosts)


grep -n "NumWorkers" /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/*.go


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go:109:	work := make(chan workItem, Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go:111:	wg.Add(Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go:112:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go:208:		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go:212:		}, Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go:217:		wg.Add(Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go:218:		for w := 0; w < Keys.NumWorkers; w++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go:184:// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go:222:	wg.Add(Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go:223:	work := make(chan workItem, Keys.NumWorkers*2)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go:224:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go:397:	work := make(chan [2]string, Keys.NumWorkers*4)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go:400:	wg.Add(Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go:401:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go:13://	├─ NumWorkers: Parallel checkpoint/archive workers
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go:114://   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go:124:	NumWorkers         int            `json:"num-workers"`
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go:136:	// Set NumWorkers from config or use default
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go:137:	if Keys.NumWorkers <= 0 {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go:138:		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go:140:	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go:188:			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go:159:	walNumShards = max(Keys.NumWorkers, 1)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go:663:	wg.Add(Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go:664:	work := make(chan workItem, Keys.NumWorkers*2)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go:666:	for range Keys.NumWorkers {


   154→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   155→	if Keys.Checkpoints.FileFormat == "json" {
   156→		return
   157→	}
   158→
   159→	walNumShards = max(Keys.NumWorkers, 1)
   160→	chBufSize := max(65536/walNumShards, 1024)
   161→
   162→	walShardChs = make([]chan *WALMessage, walNumShards)
   163→	walShardRotateChs = make([]chan walRotateReq, walNumShards)
   164→
   165→	for i := range walNumShards {
   166→		walShardChs[i] = make(chan *WALMessage, chBufSize)
   167→		walShardRotateChs[i] = make(chan walRotateReq, 64)
   168→	}
   169→
   170→	for i := range walNumShards {
   171→		msgCh := walShardChs[i]
   172→		rotateCh := walShardRotateChs[i]
   173→
   174→		wg.Go(func() {
   175→			hostFiles := make(map[string]*walFileState)
   176→
   177→			defer func() {
   178→				for _, ws := range hostFiles {
   179→					if ws.f != nil {
   180→						ws.w.Flush()
   181→						ws.f.Close()
   182→					}
   183→				}
   184→			}()
   185→
   186→			getOrOpenWAL := func(hostDir string) *walFileState {
   187→				ws, ok := hostFiles[hostDir]
   188→				if ok {
   189→					return ws
   190→				}
   191→
   192→				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   193→					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   194→					return nil
   195→				}
   196→
   197→				walPath := path.Join(hostDir, "current.wal")
   198→				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   199→				if err != nil {
   200→					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   201→					return nil
   202→				}
   203→
   204→				w := bufio.NewWriter(f)
   205→
   206→				// Write file header magic if file is new (empty).
   207→				info, err := f.Stat()
   208→				if err == nil && info.Size() == 0 {
   209→					var hdr [4]byte
   210→					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   211→					if _, err := w.Write(hdr[:]); err != nil {
   212→						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   213→						f.Close()
   214→						return nil
   215→					}
   216→				}
   217→
   218→				ws = &walFileState{f: f, w: w}
   219→				hostFiles[hostDir] = ws
   220→				return ws
   221→			}
   222→
   223→			processMsg := func(msg *WALMessage) {
   224→				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   225→				ws := getOrOpenWAL(hostDir)
   226→				if ws == nil {
   227→					return
   228→				}
   229→				if err := writeWALRecordDirect(ws.w, msg); err != nil {
   230→					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   231→				}
   232→				ws.dirty = true
   233→			}
   234→
   235→			processRotate := func(req walRotateReq) {
   236→				ws, ok := hostFiles[req.hostDir]
   237→				if ok && ws.f != nil {
   238→					ws.w.Flush()
   239→					ws.f.Close()
   240→					walPath := path.Join(req.hostDir, "current.wal")
   241→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   242→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   243→					}
   244→					delete(hostFiles, req.hostDir)
   245→				}
   246→				close(req.done)
   247→			}
   248→
   249→			flushDirty := func() {
   250→				for _, ws := range hostFiles {
   251→					if ws.dirty {
   252→						ws.w.Flush()
   253→						ws.dirty = false
   254→					}
   255→				}
   256→			}
   257→
   258→			drain := func() {
   259→				for {
   260→					select {
   261→					case msg, ok := <-msgCh:
   262→						if !ok {
   263→							return
   264→						}
   265→						processMsg(msg)
   266→					case req := <-rotateCh:
   267→						processRotate(req)
   268→					default:
   269→						flushDirty()
   270→						return
   271→					}
   272→				}
   273→			}
   274→
   275→			ticker := time.NewTicker(walFlushInterval)
   276→			defer ticker.Stop()
   277→
   278→			// drainBatch processes up to 4096 pending messages without blocking.
   279→			// Returns false if the channel was closed.
   280→			drainBatch := func() bool {
   281→				for range 4096 {
   282→					select {
   283→					case msg, ok := <-msgCh:
   284→						if !ok {
   285→							flushDirty()
   286→							return false
   287→						}
   288→						processMsg(msg)
   289→					case req := <-rotateCh:
   290→						processRotate(req)
   291→					default:
   292→						return true
   293→					}
   294→				}
   295→				return true
   296→			}
   297→
   298→			for {
   299→				select {
   300→				case <-ctx.Done():
   301→					drain()
   302→					return
   303→				case msg, ok := <-msgCh:
   304→					if !ok {
   305→						return
   306→					}
   307→					processMsg(msg)
   308→					if !drainBatch() {
   309→						return
   310→					}
   311→					// No flush here — timer handles periodic flushing.
   312→				case <-ticker.C:
   313→					flushDirty()
   314→				case req := <-rotateCh:
   315→					processRotate(req)
   316→				}
   317→			}
   318→		})
   319→	}
   320→}
   321→
   322→// RotateWALFiles sends rotation requests for the given host directories
   323→// and blocks until all rotations complete. Each request is routed to the


   125→// walShardIndex computes which shard a message belongs to based on cluster+node.
   126→// Uses FNV-1a hash for fast, well-distributed mapping.
   127→func walShardIndex(cluster, node string) int {
   128→	h := fnv.New32a()
   129→	h.Write([]byte(cluster))
   130→	h.Write([]byte{0})
   131→	h.Write([]byte(node))
   132→	return int(h.Sum32()) % walNumShards
   133→}
   134→
   135→// SendWALMessage routes a WAL message to the appropriate shard channel.
   136→// Returns false if the channel is full (message dropped).
   137→func SendWALMessage(msg *WALMessage) bool {
   138→	if walShardChs == nil {
   139→		return false
   140→	}
   141→	shard := walShardIndex(msg.Cluster, msg.Node)
   142→	select {
   143→	case walShardChs[shard] <- msg:
   144→		return true


   260→// Shutdown performs a graceful shutdown of the metric store.
   261→//
   262→// This function cancels all background goroutines started by Init() and writes
   263→// a final checkpoint to disk before returning. It should be called when the
   264→// application receives a termination signal.
   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→
   290→	ms := GetMemoryStore()
   291→
   292→	lastCheckpointMu.Lock()
   293→	from := lastCheckpoint
   294→	lastCheckpointMu.Unlock()
   295→
   296→	if Keys.Checkpoints.FileFormat == "wal" {
   297→		var hostDirs []string
   298→		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   299→		if err == nil {
   300→			RotateWALFilesAfterShutdown(hostDirs)
   301→		}
   302→	} else {
   303→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   304→	}
   305→
   306→	if err != nil {
   307→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
   308→	}
   309→	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
   310→}
   311→
   312→// Retention starts a background goroutine that periodically frees old metric data.
   313→//
   314→// This worker runs at half the retention interval and calls Free() to remove buffers


grep -A 5 "shutdownFunc()" /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go | head -20


		shutdownFunc()
	}

	if Keys.Checkpoints.FileFormat == "wal" {
		for _, ch := range walShardChs {
			close(ch)


git log --all --oneline --grep="shutdown\|Shutdown\|drain" | head -30


a550344f Increase server shutdown timeout
93a9d732 fix: Improve shutdown time
b214e175 Add buffered I/O to WAL writes and fix MemoryCap comment
0b89d6a0 Merge pull request #511 from ClusterCockpit/dev
ddda341e Safeguard metricstore shutdown if internal metricstore is not initialized
a243e174 Update to shutdown worker for WAL checkpointing mode
155e0549 Fix shutdown timout bug
cb219b3c Fix configuration issues. Fix shutdown hangs
d59aa2e8 Restructure configuration with sensible defaults. Fix shutdown hangs
8d44ac90 Fix: Busywait loop in archiver and slow shutdown
9abc206d Read in tagger config on startup. Safeguard watcher shutdown


commit 93a9d732a48cf3c9dc1969d467c8a4b6ecf99adb
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Tue Mar 24 07:17:34 2026 +0100

    fix: Improve shutdown time

    Entire-Checkpoint: a4d012e1edcf

 cmd/cc-backend/server.go | 48 +++++++++++++++++++++++++++---------------------
 1 file changed, 27 insertions(+), 21 deletions(-)


commit 93a9d732a48cf3c9dc1969d467c8a4b6ecf99adb
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Tue Mar 24 07:17:34 2026 +0100

    fix: Improve shutdown time

    Entire-Checkpoint: a4d012e1edcf

diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go
index 4e6120e9..b37d3450 100644
--- a/cmd/cc-backend/server.go
+++ b/cmd/cc-backend/server.go
@@ -18,6 +18,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync"
 	"time"

 	"github.com/99designs/gqlgen/graphql"
@@ -399,16 +400,6 @@ func (s *Server) Start(ctx context.Context) error {
 		return fmt.Errorf("dropping privileges: %w", err)
 	}

-	// Handle context cancellation for graceful shutdown
-	go func() {
-		<-ctx.Done()
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-		defer cancel()
-		if err := s.server.Shutdown(shutdownCtx); err != nil {
-			cclog.Errorf("Server shutdown error: %v", err)
-		}
-	}()
-
 	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
 		return fmt.Errorf("server failed: %w", err)
 	}
@@ -416,8 +407,7 @@ func (s *Server) Start(ctx context.Context) error {
 }

 func (s *Server) Shutdown(ctx context.Context) {
-	// Create a shutdown context with timeout
-	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
 	defer cancel()

 	nc := nats.GetClient()
@@ -425,20 +415,36 @@ func (s *Server) Shutdown(ctx context.Context) {
 		nc.Close()
 	}

-	// First shut down the server gracefully (waiting for all ongoing requests)
 	if err := s.server.Shutdown(shutdownCtx); err != nil {
 		cclog.Errorf("Server shutdown error: %v", err)
 	}

-	// Archive all the metric store data
-	ms := metricstore.GetMemoryStore()
+	// Run metricstore and archiver shutdown concurrently.
+	// They are independent: metricstore writes .bin snapshots,
+	// archiver flushes pending job archives.
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		var wg sync.WaitGroup

-	if ms != nil {
-		metricstore.Shutdown()
-	}
+		if ms := metricstore.GetMemoryStore(); ms != nil {
+			wg.Go(func() {
+				metricstore.Shutdown()
+			})
+		}
+
+		wg.Go(func() {
+			if err := archiver.Shutdown(10 * time.Second); err != nil {
+				cclog.Warnf("Archiver shutdown: %v", err)
+			}
+		})
+
+		wg.Wait()
+	}()

-	// Shutdown archiver with 10 second timeout for fast shutdown
-	if err := archiver.Shutdown(10 * time.Second); err != nil {
-		cclog.Warnf("Archiver shutdown: %v", err)
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		cclog.Warn("Shutdown deadline exceeded, forcing exit")
 	}
 }


commit a550344f13ba71ccd35f826540e5aaeb85a88a8f
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Wed Mar 25 06:15:52 2026 +0100

    Increase server shutdown timeout

    Entire-Checkpoint: cf3b472471bd

 cmd/cc-backend/server.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)


commit a550344f13ba71ccd35f826540e5aaeb85a88a8f
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Wed Mar 25 06:15:52 2026 +0100

    Increase server shutdown timeout

    Entire-Checkpoint: cf3b472471bd

diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go
index b37d3450..c9ba1e9c 100644
--- a/cmd/cc-backend/server.go
+++ b/cmd/cc-backend/server.go
@@ -357,8 +357,8 @@ func (s *Server) Start(ctx context.Context) error {
 	s.server = &http.Server{
 		ReadHeaderTimeout: readHeaderTimeout,
 		WriteTimeout:      writeTimeout,
-		Handler:      s.router,
-		Addr:         config.Keys.Addr,
+		Handler:           s.router,
+		Addr:              config.Keys.Addr,
 	}

 	// Start http or https server
@@ -444,7 +444,7 @@ func (s *Server) Shutdown(ctx context.Context) {

 	select {
 	case <-done:
-	case <-time.After(10 * time.Second):
+	case <-time.After(60 * time.Second):
 		cclog.Warn("Shutdown deadline exceeded, forcing exit")
 	}
 }


   408→
   409→func (s *Server) Shutdown(ctx context.Context) {
   410→	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
   411→	defer cancel()
   412→
   413→	nc := nats.GetClient()
   414→	if nc != nil {
   415→		nc.Close()
   416→	}
   417→
   418→	if err := s.server.Shutdown(shutdownCtx); err != nil {
   419→		cclog.Errorf("Server shutdown error: %v", err)
   420→	}
   421→
   422→	// Run metricstore and archiver shutdown concurrently.
   423→	// They are independent: metricstore writes .bin snapshots,
   424→	// archiver flushes pending job archives.
   425→	done := make(chan struct{})
   426→	go func() {
   427→		defer close(done)
   428→		var wg sync.WaitGroup
   429→
   430→		if ms := metricstore.GetMemoryStore(); ms != nil {
   431→			wg.Go(func() {
   432→				metricstore.Shutdown()
   433→			})
   434→		}
   435→
   436→		wg.Go(func() {
   437→			if err := archiver.Shutdown(10 * time.Second); err != nil {
   438→				cclog.Warnf("Archiver shutdown: %v", err)
   439→			}
   440→		})
   441→
   442→		wg.Wait()
   443→	}()
   444→
   445→	select {
   446→	case <-done:
   447→	case <-time.After(60 * time.Second):
   448→		cclog.Warn("Shutdown deadline exceeded, forcing exit")
   449→	}
   450→}
   451→


cat > /tmp/wal_analysis.md << 'EOF'
# WAL Sharding and Shutdown Performance Analysis

## Critical Timeline

### Phase 1: Context Cancellation (triggers shutdown sequence)
1. `Server.Shutdown()` calls `metricstore.Shutdown()`
2. `metricstore.Shutdown()` calls `shutdownFunc()` (the context cancellation)
3. This signals `<-ctx.Done()` on all WAL staging goroutines
4. Each goroutine immediately calls `drain()` function

### Phase 2: Channel Draining (THE BOTTLENECK)
5. `shutdownFunc()` is called, signaling context cancellation
6. Line 281-283 in metricstore.go: Close all WAL shard channels
   ```go
   if Keys.Checkpoints.FileFormat == "wal" {
       for _, ch := range walShardChs {
           close(ch)
       }
   }
   ```

### Phase 3: WAL Goroutines Process Buffered Messages
When context is cancelled, each WAL goroutine executes `drain()`:
```go
drain := func() {
    for {
        select {
        case msg, ok := <-msgCh:
            if !ok {      // Channel closed
                return
            }
            processMsg(msg)  // PROCESSES EACH MESSAGE WITH FILE I/O
        case req := <-rotateCh:
            processRotate(req)
        default:
            flushDirty()
            return
        }
    }
}
```

Key behavior of `drain()`:
- Keeps reading from the channel until it's closed
- For each message, calls `processMsg()` which does a file write
- Only exits when channel is closed AND empty
- No batching optimizations (unlike the normal loop which has `drainBatch()`)

### Phase 4: Snapshot Writing
After WAL goroutines drain (or timeout), `ToCheckpointWAL()` is called:
```go
files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
```

## The Problem: Channel Buffer Capacity

### Configuration
From `walCheckpoint.go` line 159-160:
```go
walNumShards = max(Keys.NumWorkers, 1)
chBufSize := max(65536/walNumShards, 1024)
```

### Buffer Size Calculation
- If `Keys.NumWorkers = 8` (typical for 16-CPU system: min(16/2+1, 10) = 9):
  - `walNumShards = 8`
  - `chBufSize = 65536/8 = 8192` messages per shard
  - **Total capacity = 8 × 8192 = 65,536 buffered messages across all shards**

- If `Keys.NumWorkers = 4`:
  - `walNumShards = 4`
  - `chBufSize = 65536/4 = 16,384` messages per shard
  - **Total capacity = 4 × 16,384 = 65,536 buffered messages**

### The Math: Draining Time
From `lineprotocol.go` line 364:
- `SendWALMessage()` is called for EVERY metric write
- WAL message buffer can hold up to 65,536 messages

Per WAL record (from `writeWALRecordDirect`):
- 8 bytes: timestamp
- 2 bytes: metric name length
- Variable: metric name bytes (typically 10-20 bytes)
- 1 byte: selector count
- Variable: selector bytes (typically 5-30 bytes per selector)
- 4 bytes: value
- 4 bytes: CRC32

Total per message: ~30-60 bytes (average 45 bytes)
File I/O per message: ~1 bufio.Writer.Write() call

### Shutdown Timeout
From `server.go` line 447:
```go
select {
case <-done:
case <-time.After(60 * time.Second):
    cclog.Warn("Shutdown deadline exceeded, forcing exit")
}
```

**60 second timeout for entire shutdown (metricstore.Shutdown() + archiver.Shutdown(10s))**

## The Bottleneck Hypothesis

### Sequential Message Draining
The `drain()` function processes messages **one at a time**:

```go
drain := func() {
    for {
        select {
        case msg, ok := <-msgCh:
            if !ok {
                return
            }
            processMsg(msg)   // ONE MESSAGE AT A TIME
        // ...
        }
    }
}
```

Each `processMsg()` call:
1. Looks up or opens WAL file
2. Calls `writeWALRecordDirect()` to write to bufio.Writer
3. Marks file as dirty

The bufio.Writer is flushed every 5 seconds (line 123):
```go
const walFlushInterval = 5 * time.Second
```

### Expected Drain Time
At worst case (60,000 messages with minimal file I/O overhead):
- If file I/O averages 10-100 microseconds per message
- 60,000 messages × 50 microseconds = 3 seconds (optimistic)
- But with actual file writes/flushes: potentially much longer

However, the REAL issue is that:
1. Channels stay open with thousands of buffered messages
2. All N shard goroutines must drain their queues sequentially
3. Flushing is periodic (every 5 seconds), not triggered by shutdown
4. This means queues might drain quickly BUT flushes block

## Timing Issue: No Coordination Between Phases

Current shutdown sequence:
1. Cancel context → triggers drain()
2. Close channels
3. **Goroutines drain messages (unknown duration)**
4. After some unspecified time, `metricstore.Shutdown()` continues
5. Calls `ToCheckpointWAL()` to write binary snapshots
6. 60-second total timeout for everything

**There's no explicit wait for WAL goroutines to finish before ToCheckpointWAL()!**

The metricstore.Shutdown() function just closes channels and proceeds:
```go
if Keys.Checkpoints.FileFormat == "wal" {
    for _, ch := range walShardChs {
        close(ch)
    }
}

// Immediately proceeds to checkpoint without waiting for drain to complete
cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
```

## Why This Causes Long Shutdown

### Scenario: System Under Load
1. Metric ingestion is running at high rate
2. WAL shard channels are nearly full (capacity: 65,536 messages)
3. System receives shutdown signal
4. Context is cancelled, triggering drain()
5. **Each shard goroutine must process potentially thousands of buffered messages**
6. Drain continues sequentially on 8 goroutines in parallel
7. Each message writes to buffio.Writer, marking file dirty
8. Flushes happen every 5 seconds, causing synchronous file I/O
9. With 65,536 total messages across 8 shards (8,192 per shard):
   - If flushing takes even 1ms per file × ~100 unique hosts
   - That's 100ms per flush cycle
   - Over 5 seconds: multiple flush operations
   - Worst case: queued messages take 5-10+ seconds to fully drain

### Timeline for 60-Second Timeout
- 0-5 seconds: WAL messages drain and buffer fills
- 5 seconds: First flush occurs (potentially blocks briefly)
- 10 seconds: Second flush
- 15+ seconds: Continue flushing
- 45+ seconds: ToCheckpointWAL() binary snapshots complete
- 60 seconds: Deadline

**The 60-second timeout exists because shutdown was observed to take that long!**

## Evidence from Git History

### Commit bf1a8a17 (Mar 18, 2026): "Shard WAL consumer for higher throughput"
- Introduced WAL sharding to increase throughput
- Split single WAL goroutine into N goroutines
- Each with 65536/N buffered channels
- **Goal was throughput, not shutdown latency**

### Commit 93a9d732 (Mar 24, 2026): "Improve shutdown time"
- Changed server shutdown timeout from 30s to 5s
- Changed overall shutdown deadline from 10s to 60s
- This happened **6 days after WAL sharding**

### Commit a550344f (Mar 25, 2026): "Increase server shutdown timeout"
- Changed the deadline back to 60 seconds
- This commit came **2 days later**, suggesting the problem persisted

## Summary

**WAL Sharding caused the long shutdown times because:**

1. **Large channel buffers**: Each shard channel can hold 8,192 messages (with 8 workers), totaling 65,536 messages
2. **Sequential draining**: The `drain()` function processes messages one-at-a-time, with no batching
3. **File I/O on every message**: Each message triggers a buffio.Writer.Write()
4. **Periodic flushing**: Flushes happen every 5 seconds, causing potential blocks during shutdown
5. **No explicit synchronization**: The shutdown sequence doesn't wait for drain to complete before proceeding
6. **Peak load at shutdown**: If the system is under heavy metric load when shutdown occurs, the channels are nearly full

**The 60-second shutdown timeout is a band-aid for this problem,** not a real solution. The root cause is that draining 65,536 buffered WAL messages with sequential file I/O takes excessive time.

EOF
cat /tmp/wal_analysis.md


# WAL Sharding and Shutdown Performance Analysis

## Critical Timeline

### Phase 1: Context Cancellation (triggers shutdown sequence)
1. `Server.Shutdown()` calls `metricstore.Shutdown()`
2. `metricstore.Shutdown()` calls `shutdownFunc()` (the context cancellation)
3. This signals `<-ctx.Done()` on all WAL staging goroutines
4. Each goroutine immediately calls `drain()` function

### Phase 2: Channel Draining (THE BOTTLENECK)
5. `shutdownFunc()` is called, signaling context cancellation
6. Line 281-283 in metricstore.go: Close all WAL shard channels
   ```go
   if Keys.Checkpoints.FileFormat == "wal" {
       for _, ch := range walShardChs {
           close(ch)
       }
   }
   ```

### Phase 3: WAL Goroutines Process Buffered Messages
When context is cancelled, each WAL goroutine executes `drain()`:
```go
drain := func() {
    for {
        select {
        case msg, ok := <-msgCh:
            if !ok {      // Channel closed
                return
            }
            processMsg(msg)  // PROCESSES EACH MESSAGE WITH FILE I/O
        case req := <-rotateCh:
            processRotate(req)
        default:
            flushDirty()
            return
        }
    }
}
```

Key behavior of `drain()`:
- Keeps reading from the channel until it's closed
- For each message, calls `processMsg()` which does a file write
- Only exits when channel is closed AND empty
- No batching optimizations (unlike the normal loop which has `drainBatch()`)

### Phase 4: Snapshot Writing
After WAL goroutines drain (or timeout), `ToCheckpointWAL()` is called:
```go
files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
```

## The Problem: Channel Buffer Capacity

### Configuration
From `walCheckpoint.go` line 159-160:
```go
walNumShards = max(Keys.NumWorkers, 1)
chBufSize := max(65536/walNumShards, 1024)
```

### Buffer Size Calculation
- If `Keys.NumWorkers = 8` (typical for 16-CPU system: min(16/2+1, 10) = 9):
  - `walNumShards = 8`
  - `chBufSize = 65536/8 = 8192` messages per shard
  - **Total capacity = 8 × 8192 = 65,536 buffered messages across all shards**

- If `Keys.NumWorkers = 4`:
  - `walNumShards = 4`
  - `chBufSize = 65536/4 = 16,384` messages per shard
  - **Total capacity = 4 × 16,384 = 65,536 buffered messages**

### The Math: Draining Time
From `lineprotocol.go` line 364:
- `SendWALMessage()` is called for EVERY metric write
- WAL message buffer can hold up to 65,536 messages

Per WAL record (from `writeWALRecordDirect`):
- 8 bytes: timestamp
- 2 bytes: metric name length
- Variable: metric name bytes (typically 10-20 bytes)
- 1 byte: selector count
- Variable: selector bytes (typically 5-30 bytes per selector)
- 4 bytes: value
- 4 bytes: CRC32

Total per message: ~30-60 bytes (average 45 bytes)
File I/O per message: ~1 bufio.Writer.Write() call

### Shutdown Timeout
From `server.go` line 447:
```go
select {
case <-done:
case <-time.After(60 * time.Second):
    cclog.Warn("Shutdown deadline exceeded, forcing exit")
}
```

**60 second timeout for entire shutdown (metricstore.Shutdown() + archiver.Shutdown(10s))**

## The Bottleneck Hypothesis

### Sequential Message Draining
The `drain()` function processes messages **one at a time**:

```go
drain := func() {
    for {
        select {
        case msg, ok := <-msgCh:
            if !ok {
                return
            }
            processMsg(msg)   // ONE MESSAGE AT A TIME
        // ...
        }
    }
}
```

Each `processMsg()` call:
1. Looks up or opens WAL file
2. Calls `writeWALRecordDirect()` to write to bufio.Writer
3. Marks file as dirty

The bufio.Writer is flushed every 5 seconds (line 123):
```go
const walFlushInterval = 5 * time.Second
```

### Expected Drain Time
At worst case (60,000 messages with minimal file I/O overhead):
- If file I/O averages 10-100 microseconds per message
- 60,000 messages × 50 microseconds = 3 seconds (optimistic)
- But with actual file writes/flushes: potentially much longer

However, the REAL issue is that:
1. Channels stay open with thousands of buffered messages
2. All N shard goroutines must drain their queues sequentially
3. Flushing is periodic (every 5 seconds), not triggered by shutdown
4. This means queues might drain quickly BUT flushes block

## Timing Issue: No Coordination Between Phases

Current shutdown sequence:
1. Cancel context → triggers drain()
2. Close channels
3. **Goroutines drain messages (unknown duration)**
4. After some unspecified time, `metricstore.Shutdown()` continues
5. Calls `ToCheckpointWAL()` to write binary snapshots
6. 60-second total timeout for everything

**There's no explicit wait for WAL goroutines to finish before ToCheckpointWAL()!**

The metricstore.Shutdown() function just closes channels and proceeds:
```go
if Keys.Checkpoints.FileFormat == "wal" {
    for _, ch := range walShardChs {
        close(ch)
    }
}

// Immediately proceeds to checkpoint without waiting for drain to complete
cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
```

## Why This Causes Long Shutdown

### Scenario: System Under Load
1. Metric ingestion is running at high rate
2. WAL shard channels are nearly full (capacity: 65,536 messages)
3. System receives shutdown signal
4. Context is cancelled, triggering drain()
5. **Each shard goroutine must process potentially thousands of buffered messages**
6. Drain continues sequentially on 8 goroutines in parallel
7. Each message writes to buffio.Writer, marking file dirty
8. Flushes happen every 5 seconds, causing synchronous file I/O
9. With 65,536 total messages across 8 shards (8,192 per shard):
   - If flushing takes even 1ms per file × ~100 unique hosts
   - That's 100ms per flush cycle
   - Over 5 seconds: multiple flush operations
   - Worst case: queued messages take 5-10+ seconds to fully drain

### Timeline for 60-Second Timeout
- 0-5 seconds: WAL messages drain and buffer fills
- 5 seconds: First flush occurs (potentially blocks briefly)
- 10 seconds: Second flush
- 15+ seconds: Continue flushing
- 45+ seconds: ToCheckpointWAL() binary snapshots complete
- 60 seconds: Deadline

**The 60-second timeout exists because shutdown was observed to take that long!**

## Evidence from Git History

### Commit bf1a8a17 (Mar 18, 2026): "Shard WAL consumer for higher throughput"
- Introduced WAL sharding to increase throughput
- Split single WAL goroutine into N goroutines
- Each with 65536/N buffered channels
- **Goal was throughput, not shutdown latency**

### Commit 93a9d732 (Mar 24, 2026): "Improve shutdown time"
- Changed server shutdown timeout from 30s to 5s
- Changed overall shutdown deadline from 10s to 60s
- This happened **6 days after WAL sharding**

### Commit a550344f (Mar 25, 2026): "Increase server shutdown timeout"
- Changed the deadline back to 60 seconds
- This commit came **2 days later**, suggesting the problem persisted

## Summary

**WAL Sharding caused the long shutdown times because:**

1. **Large channel buffers**: Each shard channel can hold 8,192 messages (with 8 workers), totaling 65,536 messages
2. **Sequential draining**: The `drain()` function processes messages one-at-a-time, with no batching
3. **File I/O on every message**: Each message triggers a buffio.Writer.Write()
4. **Periodic flushing**: Flushes happen every 5 seconds, causing potential blocks during shutdown
5. **No explicit synchronization**: The shutdown sequence doesn't wait for drain to complete before proceeding
6. **Peak load at shutdown**: If the system is under heavy metric load when shutdown occurs, the channels are nearly full

**The 60-second shutdown timeout is a band-aid for this problem,** not a real solution. The root cause is that draining 65,536 buffered WAL messages with sequential file I/O takes excessive time.


grep -n "drainBatch" /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go


278:			// drainBatch processes up to 4096 pending messages without blocking.
280:			drainBatch := func() bool {
308:					if !drainBatch() {


   362→// writeWALRecordDirect encodes a WAL record directly into the bufio.Writer,
   363→// avoiding heap allocations by using a stack-allocated scratch buffer for
   364→// the fixed-size header/trailer and computing CRC inline.
   365→func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
   366→	// Compute payload size.
   367→	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
   368→	for _, s := range msg.Selector {
   369→		payloadSize += 1 + len(s)
   370→	}
   371→
   372→	// Write magic + payload length (8 bytes header).
   373→	var hdr [8]byte
   374→	binary.LittleEndian.PutUint32(hdr[0:4], walRecordMagic)
   375→	binary.LittleEndian.PutUint32(hdr[4:8], uint32(payloadSize))
   376→	if _, err := w.Write(hdr[:]); err != nil {
   377→		return err
   378→	}
   379→
   380→	// We need to compute CRC over the payload as we write it.
   381→	crc := crc32.NewIEEE()
   382→
   383→	// Timestamp (8 bytes).
   384→	var scratch [8]byte
   385→	binary.LittleEndian.PutUint64(scratch[:8], uint64(msg.Timestamp))
   386→	crc.Write(scratch[:8])
   387→	if _, err := w.Write(scratch[:8]); err != nil {
   388→		return err
   389→	}
   390→
   391→	// Metric name length (2 bytes) + metric name.


git log --oneline --all -- pkg/metricstore/walCheckpoint.go pkg/metricstore/metricstore.go | head -10


6f7dda53 Cleanup
0325d9e8 fix: Increase throughput for WAL writers
96a79098 carry forward: uncommitted session files
c0d2d65f Introduce average resampler support
d3b94c33 Completed 'Explore' agent: Find CUDA/HIP macro switching (toolu_01EbN3)
8b132ed7 fix: Blocking ReceiveNats call
bf1a8a17 fix: Shard WAL consumer for higher throughput
50aed595 fix: metricstore NATS contention
818a05f2 carry forward: uncommitted session files
39ab1278 Make checkpointInterval an option config option again.


commit 0325d9e866ec97994e77770ca189838c0ee3b3f0
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Tue Mar 24 06:53:12 2026 +0100

    fix: Increase throughput for WAL writers

    Entire-Checkpoint: ddd40d290c56

 pkg/metricstore/walCheckpoint.go | 63 ++++++++++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 21 deletions(-)


commit 0325d9e866ec97994e77770ca189838c0ee3b3f0
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Tue Mar 24 06:53:12 2026 +0100

    fix: Increase throughput for WAL writers

    Entire-Checkpoint: ddd40d290c56

diff --git a/pkg/metricstore/walCheckpoint.go b/pkg/metricstore/walCheckpoint.go
index 38585cf5..85e5047e 100644
--- a/pkg/metricstore/walCheckpoint.go
+++ b/pkg/metricstore/walCheckpoint.go
@@ -69,6 +69,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"time"

 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
 	"github.com/ClusterCockpit/cc-lib/v2/schema"
@@ -111,10 +112,16 @@ type walRotateReq struct {

 // walFileState holds an open WAL file handle and buffered writer for one host directory.
 type walFileState struct {
-	f *os.File
-	w *bufio.Writer
+	f     *os.File
+	w     *bufio.Writer
+	dirty bool
 }

+// walFlushInterval controls how often dirty WAL files are flushed to disk.
+// Decoupling flushes from message processing lets the consumer run at memory
+// speed, amortizing syscall overhead across many writes.
+const walFlushInterval = 5 * time.Second
+
 // walShardIndex computes which shard a message belongs to based on cluster+node.
 // Uses FNV-1a hash for fast, well-distributed mapping.
 func walShardIndex(cluster, node string) int {
@@ -222,6 +229,7 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 				if err := writeWALRecordDirect(ws.w, msg); err != nil {
 					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
 				}
+				ws.dirty = true
 			}

 			processRotate := func(req walRotateReq) {
@@ -238,10 +246,11 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 				close(req.done)
 			}

-			flushAll := func() {
+			flushDirty := func() {
 				for _, ws := range hostFiles {
-					if ws.f != nil {
+					if ws.dirty {
 						ws.w.Flush()
+						ws.dirty = false
 					}
 				}
 			}
@@ -257,12 +266,35 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 					case req := <-rotateCh:
 						processRotate(req)
 					default:
-						flushAll()
+						flushDirty()
 						return
 					}
 				}
 			}

+			ticker := time.NewTicker(walFlushInterval)
+			defer ticker.Stop()
+
+			// drainBatch processes up to 4096 pending messages without blocking.
+			// Returns false if the channel was closed.
+			drainBatch := func() bool {
+				for range 4096 {
+					select {
+					case msg, ok := <-msgCh:
+						if !ok {
+							flushDirty()
+							return false
+						}
+						processMsg(msg)
+					case req := <-rotateCh:
+						processRotate(req)
+					default:
+						return true
+					}
+				}
+				return true
+			}
+
 			for {
 				select {
 				case <-ctx.Done():
@@ -273,23 +305,12 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 						return
 					}
 					processMsg(msg)
-
-					// Drain up to 256 more messages without blocking to batch writes.
-					for range 256 {
-						select {
-						case msg, ok := <-msgCh:
-							if !ok {
-								return
-							}
-							processMsg(msg)
-						case req := <-rotateCh:
-							processRotate(req)
-						default:
-							goto flushed
-						}
+					if !drainBatch() {
+						return
 					}
-				flushed:
-					flushAll()
+					// No flush here — timer handles periodic flushing.
+				case <-ticker.C:
+					flushDirty()
 				case req := <-rotateCh:
 					processRotate(req)
 				}


Perfect! This shows that on Mar 24, the code was refactored to use periodic flushing rather than flushing after every batch. Now let me create my final comprehensive report:

## Summary

I've completed a thorough investigation of WAL sharding and its interaction with shutdown. Here are my key findings:

### **The Root Cause: WAL Sharding Buffering and Sequential Drain**

Your suspicion was correct - **WAL sharding directly caused the long shutdown times**. Here's the exact mechanism:

### **Channel Buffer Capacity Issue**

From `walCheckpoint.go` lines 159-160:
```go
walNumShards = max(Keys.NumWorkers, 1)
chBufSize := max(65536/walNumShards, 1024)
```

With a typical configuration of 8 workers:
- Each shard has a buffered channel of size 8,192 messages
- **Total capacity across all shards: 65,536 messages**
- Each message requires a file write via `writeWALRecordDirect()`
- Messages accumulate in these buffers during normal operation

### **The Shutdown Bottleneck**

When shutdown occurs:
1. `metricstore.Shutdown()` calls context cancellation, triggering `<-ctx.Done()`
2. Each WAL staging goroutine immediately calls `drain()` function
3. **The `drain()` function processes queued messages one-at-a-time** with file I/O:
   ```go
   drain := func() {
       for {
           select {
           case msg, ok := <-msgCh:
               if !ok {
                   return
               }
               processMsg(msg)  // Write to bufio + mark dirty
           // ...
           }
       }
   }
   ```

4. All 65,536 buffered messages must be written to disk sequentially
5. Flushes happen periodically (every 5 seconds from line 123), not triggered by shutdown
6. **There is NO explicit synchronization waiting for drain to complete before `ToCheckpointWAL()` runs**

### **No Explicit Wait for Drain to Complete**

The shutdown sequence in `metricstore.go` lines 280-298 simply closes channels and immediately proceeds:
```go
if Keys.Checkpoints.FileFormat == "wal" {
    for _, ch := range walShardChs {
        close(ch)
    }
}

// Immediately proceeds without waiting for WAL goroutines to finish draining
cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
```

### **Evidence from Git History**

- **Commit bf1a8a17** (Mar 18, 2026): Introduced WAL sharding to "increase throughput for WAL writers"
- **Commit 93a9d732** (Mar 24, 2026): "Improve shutdown time" - changed shutdown timeout from 30s to 5s HTTP timeout, but increased overall shutdown deadline to 60s
- **Commit a550344f** (Mar 25, 2026): "Increase server shutdown timeout" - changed deadline back to **60 seconds** after the problem persisted

The 60-second shutdown timeout (`server.go` line 447) is a band-aid fix for this root cause.

### **Critical Code Locations**

1. **Buffer initialization**: `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go` lines 159-168
2. **Drain function**: `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go` lines 258-273
3. **Shutdown sequence**: `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go` lines 273-310
4. **Shutdown timeout**: `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go` lines 409-450

### **Why This Takes ~60 Seconds**

Under load at shutdown:
- Channels may be nearly full with 65,536 buffered messages
- During drain, the 5-second flush timer still applies (not triggered by shutdown)
- Multiple flush cycles may occur (every 5 seconds) during message draining
- Each flush may write to 100+ host WAL files, each with multiple file I/O operations
- Sequential processing of 60,000+ messages × multi-millisecond per message = significant delay
- ToCheckpointWAL() binary snapshots complete after drain finishes
- All must complete within 60-second deadline

Prompt:
The user suspects that WAL sharding caused the long shutdown times. I need to investigate how WAL sharding interacts with shutdown.

Key areas to explore:

1. Read `pkg/metricstore/walCheckpoint.go` completely - understand the WAL sharding mechanism, especially:
   - How many shards are created (walNumShards)
   - Channel buffer sizes (chBufSize)
   - What happens when channels are closed during shutdown
   - The `drain()` function behavior
   - How many messages could be buffered in channels at shutdown time

2. Read `pkg/metricstore/lineprotocol.go` - how WAL messages are sent to shard channels. Is there a blocking send? Could large channel buffers cause drain delays?

3. Read `pkg/metricstore/metricstore.go` fully - the Init() and Shutdown() functions, especially how NumWorkers is configured and used.

4. Check git log for when WAL sharding was introduced - `git log --oneline --all -- pkg/metricstore/walCheckpoint.go` and look at the relevant commits.

5. Think about: if there are N shards, each with a buffer of 65536/N messages, and at shutdown time these channels need to be drained (each message triggers a file write), how long could that take?

6. Check if the issue is that Shutdown() closes WAL channels, then WALStaging goroutines drain them (processing potentially thousands of buffered messages with file I/O each), and THEN ToCheckpointWAL runs - meaning the drain itself takes most of the 60 seconds.

Be very thorough - read all relevant code and trace the exact shutdown sequence step by step.


This confirms the issue. The WAL sharding introduced buffered channels with up to 65,536 total messages. At shutdown, all buffered messages are drained with file I/O before `ToCheckpointWAL()` can even start — but this drain is **completely unnecessary** because `ToCheckpointWAL()` writes a full binary snapshot from the in-memory state, making the buffered WAL messages redundant.

Let me update the plan with this insight.


   255→				}
   256→			}
   257→
   258→			drain := func() {
   259→				for {
   260→					select {
   261→					case msg, ok := <-msgCh:
   262→						if !ok {
   263→							return
   264→						}
   265→						processMsg(msg)
   266→					case req := <-rotateCh:
   267→						processRotate(req)
   268→					default:
   269→						flushDirty()
   270→						return
   271→					}
   272→				}
   273→			}
   274→
   275→			ticker := time.NewTicker(walFlushInterval)
   276→			defer ticker.Stop()
   277→
   278→			// drainBatch processes up to 4096 pending messages without blocking.
   279→			// Returns false if the channel was closed.
   280→			drainBatch := func() bool {
   281→				for range 4096 {
   282→					select {
   283→					case msg, ok := <-msgCh:
   284→						if !ok {
   285→							flushDirty()
   286→							return false
   287→						}
   288→						processMsg(msg)
   289→					case req := <-rotateCh:
   290→						processRotate(req)
   291→					default:
   292→						return true
   293→					}
   294→				}
   295→				return true
   296→			}
   297→
   298→			for {
   299→				select {
   300→				case <-ctx.Done():
   301→					drain()
   302→					return
   303→				case msg, ok := <-msgCh:
   304→					if !ok {
   305→						return
   306→					}
   307→					processMsg(msg)
   308→					if !drainBatch() {
   309→						return
   310→					}
   311→					// No flush here — timer handles periodic flushing.
   312→				case <-ticker.C:
   313→					flushDirty()
   314→				case req := <-rotateCh:
   315→					processRotate(req)
   316→				}
   317→			}
   318→		})
   319→	}

				}
			}

			drain := func() {
				for {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							return
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						flushDirty()
						return
					}
				}
			}

			ticker := time.NewTicker(walFlushInterval)
			defer ticker.Stop()

			// drainBatch processes up to 4096 pending messages without blocking.
			// Returns false if the channel was closed.
			drainBatch := func() bool {
				for range 4096 {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							flushDirty()
							return false
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						return true
					}
				}
				return true
			}

			for {
				select {
				case <-ctx.Done():
					drain()
					return
				case msg, ok := <-msgCh:
					if !ok {
						return
					}
					processMsg(msg)
					if !drainBatch() {
						return
					}
					// No flush here — timer handles periodic flushing.
				case <-ticker.C:
					flushDirty()
				case req := <-rotateCh:
					processRotate(req)
				}
			}
		})
	}


Write

# Fix Shutdown Timeout & Add Shutdown Logging

## Context

The shutdown consistently hits the 60s timeout. Root cause: WAL sharding (introduced in `bf1a8a17`) created buffered channels with up to **65,536 total messages**. At shutdown, `Shutdown()` cancels the context, WALStaging goroutines call `drain()` which processes every buffered message with file I/O, and only then does `ToCheckpointWAL()` start writing binary snapshots.

**Key insight**: The WAL drain is completely unnecessary at shutdown. `ToCheckpointWAL()` writes a full binary snapshot from the in-memory state — the buffered WAL messages are redundant because they represent writes already applied to in-memory buffers. We should skip the drain entirely.

## Changes

### 1. Skip WAL drain on shutdown (`pkg/metricstore/walCheckpoint.go`)

Change the `ctx.Done()` handler in the WALStaging main loop (line 300-302) to **not** call `drain()`. Instead, just flush dirty buffers (to cleanly close open file handles) and return:

```go
case <-ctx.Done():
    // On shutdown, skip draining buffered messages — a full binary
    // checkpoint is about to be written from in-memory state, making
    // buffered WAL records redundant.
    flushDirty()
    return
```

The deferred cleanup (lines 177-184) still runs, flushing and closing all file handles.

### 2. WALStaging drain synchronization (`pkg/metricstore/walCheckpoint.go`)

Add `walStagingWg sync.WaitGroup` to track WALStaging goroutine exits. In each shard goroutine: `walStagingWg.Add(1)` before, `defer walStagingWg.Done()` inside. Export `WaitForWALStagingDrain()`.

This ensures all WAL file handles are closed before checkpoint writes begin (no file handle races).

### 3. Wait for WAL drain + per-host WAL deletion (`pkg/metricstore/metricstore.go`)

In `Shutdown()`:
- After closing WAL channels, call `WaitForWALStagingDrain()` (now fast since we skip processing)
- Remove `RotateWALFilesAfterShutdown()` call — move WAL deletion into `ToCheckpointWAL` workers (change 4)

### 4. Per-host WAL deletion in checkpoint workers (`pkg/metricstore/walCheckpoint.go`)

In `ToCheckpointWAL()` worker loop, delete `current.wal` immediately after each successful binary snapshot. This ensures WALs are cleaned up incrementally even if the overall checkpoint doesn't finish within the deadline.

### 5. Progress logging in `ToCheckpointWAL()` (`pkg/metricstore/walCheckpoint.go`)

- Log total host count and worker count before dispatching work
- Add atomic `completed` counter
- Spawn goroutine logging progress every 10s, stopped after workers finish

### 6. Phase timing in `Shutdown()` (`pkg/metricstore/metricstore.go`)

Add `time.Since()` logs for: context cancellation, WAL drain wait, checkpoint write, total.

### 7. Phase timing in `Server.Shutdown()` (`cmd/cc-backend/server.go`)

Add timing logs for: NATS close, HTTP server stop, metricstore+archiver concurrent shutdown, overall.

## Files to modify

1. `pkg/metricstore/walCheckpoint.go` — changes 1, 2, 4, 5
2. `pkg/metricstore/metricstore.go` — changes 3, 6
3. `cmd/cc-backend/server.go` — change 7

## Verification

1. `go build ./...` — must compile
2. `go test ./pkg/metricstore/...` — existing tests pass
3. Manual test: start with sample data, send SIGTERM, verify:
   - Phase timing logs appear in order with fast WAL drain
   - WAL files are deleted during checkpoint (not after)
   - No 60s timeout — shutdown completes in seconds