     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides an efficient in-memory time-series metric storage system
     7→// with support for hierarchical data organization, checkpointing, and archiving.
     8→//
     9→// The package organizes metrics in a tree structure (cluster → host → component) and
    10→// provides concurrent read/write access to metric data with configurable aggregation strategies.
    11→// Background goroutines handle periodic checkpointing (JSON or WAL/binary format), archiving old data,
    12→// and enforcing retention policies.
    13→//
    14→// Key features:
    15→//   - In-memory metric storage with configurable retention
    16→//   - Hierarchical data organization (selectors)
    17→//   - Concurrent checkpoint/archive workers
    18→//   - Support for sum and average aggregation
    19→//   - NATS integration for metric ingestion via InfluxDB line protocol
    20→package metricstore
    21→
    22→import (
    23→	"bytes"
    24→	"context"
    25→	"encoding/json"
    26→	"errors"
    27→	"fmt"
    28→	"runtime"
    29→	"runtime/debug"
    30→	"slices"
    31→	"sync"
    32→	"time"
    33→
    34→	"github.com/ClusterCockpit/cc-backend/internal/config"
    35→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    36→	"github.com/ClusterCockpit/cc-lib/v2/resampler"
    37→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    38→	"github.com/ClusterCockpit/cc-lib/v2/util"
    39→)
    40→
    41→// GlobalState holds the global state for the metric store with thread-safe access.
    42→type GlobalState struct {
    43→	mu                sync.RWMutex
    44→	lastRetentionTime int64
    45→	selectorsExcluded bool
    46→}
    47→
    48→var (
    49→	singleton  sync.Once
    50→	msInstance *MemoryStore
    51→	// shutdownFunc stores the context cancellation function created in Init
    52→	// and is called during Shutdown to cancel all background goroutines
    53→	shutdownFunc   context.CancelFunc
    54→	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
    55→	// Create a global instance
    56→	state = &GlobalState{}
    57→)
    58→
    59→// NodeProvider provides information about nodes currently in use by running jobs.
    60→//
    61→// This interface allows metricstore to query job information without directly
    62→// depending on the repository package, breaking the import cycle.
    63→//
    64→// Implementations should return nodes that are actively processing jobs started
    65→// before the given timestamp. These nodes will be excluded from retention-based
    66→// garbage collection to prevent data loss for jobs that are still running or
    67→// recently completed.
    68→type NodeProvider interface {
    69→	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
    70→	// that are currently in use by jobs that started before the given timestamp.
    71→	//
    72→	// Parameters:
    73→	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
    74→	//
    75→	// Returns:
    76→	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
    77→	//   - Error if the query fails
    78→	GetUsedNodes(ts int64) (map[string][]string, error)
    79→}
    80→
    81→// Metric represents a single metric data point to be written to the store.
    82→type Metric struct {
    83→	Name  string
    84→	Value schema.Float
    85→	// MetricConfig contains frequency and aggregation settings for this metric.
    86→	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
    87→	MetricConfig MetricConfig
    88→}
    89→
    90→// MemoryStore is the main in-memory time-series metric storage implementation.
    91→//
    92→// It organizes metrics in a hierarchical tree structure where each level represents
    93→// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
    94→// store multiple metrics as time-series buffers.
    95→//
    96→// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
    97→// All public methods are safe for concurrent use.
    98→type MemoryStore struct {
    99→	Metrics      map[string]MetricConfig
   100→	root         Level
   101→	nodeProvider NodeProvider
   102→}
   103→
   104→// Init initializes the metric store from configuration and starts background workers.
   105→//
   106→// This function must be called exactly once before any other metricstore operations.
   107→// It performs the following initialization steps:
   108→//  1. Validates and decodes the metric store configuration
   109→//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
   110→//  3. Loads metric configurations from all registered clusters
   111→//  4. Restores checkpoints within the retention window
   112→//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
   113→//  6. Optionally subscribes to NATS for real-time metric ingestion
   114→//
   115→// Parameters:
   116→//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig); may be nil to use defaults
   117→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
   118→//   - wg: WaitGroup that will be incremented for each background goroutine started
   119→//
   120→// The function will call cclog.Fatal on critical errors during initialization.
   121→// Use Shutdown() to cleanly stop all background workers started by Init().
   122→//
   123→// Note: Signal handling must be implemented by the caller. Call Shutdown() when
   124→// receiving termination signals to ensure checkpoint data is persisted.
   125→func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
   126→	startupTime := time.Now()
   127→
   128→	if rawConfig != nil {
   129→		config.Validate(configSchema, rawConfig)
   130→		dec := json.NewDecoder(bytes.NewReader(rawConfig))
   131→		dec.DisallowUnknownFields()
   132→		if err := dec.Decode(&Keys); err != nil {
   133→			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
   134→		}
   135→	}
   136→
   137→	// Set NumWorkers from config or use default
   138→	if Keys.NumWorkers <= 0 {
   139→		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
   140→	}
   141→	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
   142→
   143→	// Pass the config.MetricStoreKeys
   144→	InitMetrics(metrics)
   145→
   146→	ms := GetMemoryStore()
   147→
   148→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   149→	if err != nil {
   150→		cclog.Fatal(err)
   151→	}
   152→
   153→	restoreFrom := startupTime.Add(-d)
   154→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   155→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   156→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   157→	if err != nil {
   158→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   159→	} else {
   160→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   161→	}
   162→
   163→	// Try to use less memory by forcing a GC run here and then
   164→	// lowering the target percentage. The default of 100 means
   165→	// that only once the ratio of new allocations execeds the
   166→	// previously active heap, a GC is triggered.
   167→	// Forcing a GC here will set the "previously active heap"
   168→	// to a minumum.
   169→	// runtime.GC()
   170→
   171→	ctx, shutdown := context.WithCancel(context.Background())
   172→
   173→	Retention(wg, ctx)
   174→	Checkpointing(wg, ctx)
   175→	CleanUp(wg, ctx)
   176→	WALStaging(wg, ctx)
   177→	MemoryUsageTracker(wg, ctx)
   178→
   179→	// Note: Signal handling has been removed from this function.
   180→	// The caller is responsible for handling shutdown signals and calling
   181→	// the shutdown() function when appropriate.
   182→	// Store the shutdown function for later use by Shutdown()
   183→	shutdownFuncMu.Lock()
   184→	shutdownFunc = shutdown
   185→	shutdownFuncMu.Unlock()
   186→
   187→	if Keys.Subscriptions != nil {
   188→		wg.Go(func() {
   189→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   190→				cclog.Fatal(err)
   191→			}
   192→		})
   193→	}
   194→}
   195→
   196→// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
   197→//
   198→// This function must be called before GetMemoryStore() and can only be called once due to
   199→// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
   200→//
   201→// Parameters:
   202→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
   203→//
   204→// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
   205→//
   206→// After this call, the global msInstance is ready for use via GetMemoryStore().
   207→func InitMetrics(metrics map[string]MetricConfig) {
   208→	singleton.Do(func() {
   209→		offset := 0
   210→		for key, cfg := range metrics {
   211→			if cfg.Frequency == 0 {
   212→				panic("[METRICSTORE]> invalid frequency")
   213→			}
   214→
   215→			metrics[key] = MetricConfig{
   216→				Frequency:   cfg.Frequency,
   217→				Aggregation: cfg.Aggregation,
   218→				offset:      offset,
   219→			}
   220→			offset += 1
   221→		}
   222→
   223→		msInstance = &MemoryStore{
   224→			root: Level{
   225→				metrics:  make([]*buffer, len(metrics)),
   226→				children: make(map[string]*Level),
   227→			},
   228→			Metrics: metrics,
   229→		}
   230→	})
   231→}
   232→
   233→// GetMemoryStore returns the singleton MemoryStore instance.
   234→//
   235→// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
   236→//
   237→// This function is safe for concurrent use after initialization.
   238→func GetMemoryStore() *MemoryStore {
   239→	if msInstance == nil {
   240→		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
   241→	}
   242→
   243→	return msInstance
   244→}
   245→
   246→func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
   247→	if metric, ok := ms.Metrics[metricName]; ok {
   248→		return metric.Frequency, nil
   249→	}
   250→	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
   251→}
   252→
   253→// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
   254→// This must be called during initialization to provide job state information
   255→// for selective buffer retention during Free operations.
   256→// If not set, the Free function will fall back to freeing all buffers.
   257→func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
   258→	ms.nodeProvider = provider
   259→}
   260→
   261→// Shutdown performs a graceful shutdown of the metric store.
   262→//
   263→// This function cancels all background goroutines started by Init() and writes
   264→// a final checkpoint to disk before returning. It should be called when the
   265→// application receives a termination signal.
   266→//
   267→// The function will:
   268→//  1. Cancel the context to stop all background workers
   269→//  2. Close the WAL messages channel if using WAL format
   270→//  3. Write a final checkpoint to preserve in-memory data
   271→//  4. Log any errors encountered during shutdown
   272→//
   273→// Note: This function blocks until the final checkpoint is written.
   274→func Shutdown() {
   275→	totalStart := time.Now()
   276→
   277→	shutdownFuncMu.Lock()
   278→	defer shutdownFuncMu.Unlock()
   279→	if shutdownFunc != nil {
   280→		shutdownFunc()
   281→	}
   282→	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))
   283→
   284→	if Keys.Checkpoints.FileFormat == "wal" {
   285→		// Signal producers to stop sending before closing channels,
   286→		// preventing send-on-closed-channel panics from in-flight NATS workers.
   287→		walShuttingDown.Store(true)
   288→		// Brief grace period for in-flight DecodeLine calls to complete.
   289→		time.Sleep(100 * time.Millisecond)
   290→
   291→		for _, ch := range walShardChs {
   292→			close(ch)
   293→		}
   294→		drainStart := time.Now()
   295→		WaitForWALStagingDrain()
   296→		cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
   297→	}
   298→
   299→	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
   300→	checkpointStart := time.Now()
   301→	var files int
   302→	var err error
   303→
   304→	ms := GetMemoryStore()
   305→
   306→	lastCheckpointMu.Lock()
   307→	from := lastCheckpoint
   308→	lastCheckpointMu.Unlock()
   309→
   310→	if Keys.Checkpoints.FileFormat == "wal" {
   311→		// WAL files are deleted per-host inside ToCheckpointWAL workers.
   312→		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   313→	} else {
   314→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   315→	}
   316→
   317→	if err != nil {
   318→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
   319→	}
   320→	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
   321→}
   322→
   323→// Retention starts a background goroutine that periodically frees old metric data.
   324→//
   325→// This worker runs at half the retention interval and calls Free() to remove buffers
   326→// older than the configured retention time. It respects the NodeProvider to preserve
   327→// data for nodes with active jobs.
   328→//
   329→// Parameters:
   330→//   - wg: WaitGroup to signal completion when context is cancelled
   331→//   - ctx: Context for cancellation signal
   332→//
   333→// The goroutine exits when ctx is cancelled.
   334→func Retention(wg *sync.WaitGroup, ctx context.Context) {
   335→	ms := GetMemoryStore()
   336→
   337→	wg.Go(func() {
   338→		d, err := time.ParseDuration(Keys.RetentionInMemory)
   339→		if err != nil {
   340→			cclog.Fatal(err)
   341→		}
   342→		if d <= 0 {
   343→			return
   344→		}
   345→
   346→		tickInterval := d / 2
   347→		if tickInterval <= 0 {
   348→			return
   349→		}
   350→		ticker := time.NewTicker(tickInterval)
   351→		defer ticker.Stop()
   352→
   353→		for {
   354→			select {
   355→			case <-ctx.Done():
   356→				return
   357→			case <-ticker.C:
   358→				state.mu.Lock()
   359→
   360→				t := time.Now().Add(-d)
   361→
   362→				state.lastRetentionTime = t.Unix()
   363→
   364→				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
   365→
   366→				freed, err := Free(ms, t)
   367→				if err != nil {
   368→					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
   369→				} else {
   370→					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
   371→				}
   372→
   373→				state.mu.Unlock()
   374→
   375→				// Clean up the buffer pool
   376→				bufferPool.Clean(state.lastRetentionTime)
   377→			}
   378→		}
   379→	})
   380→}
   381→
   382→// MemoryUsageTracker starts a background goroutine that monitors memory usage.
   383→//
   384→// This worker checks actual process memory usage (via runtime.MemStats) periodically
   385→// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
   386→// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
   387→// performance issues.
   388→//
   389→// The tracker logs both actual memory usage (heap allocated) and metric data size for
   390→// visibility into memory overhead from Go runtime structures and allocations.
   391→//
   392→// Parameters:
   393→//   - wg: WaitGroup to signal completion when context is cancelled
   394→//   - ctx: Context for cancellation signal
   395→//
   396→// The goroutine exits when ctx is cancelled.
   397→func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
   398→	ms := GetMemoryStore()
   399→
   400→	wg.Go(func() {
   401→		normalInterval := DefaultMemoryUsageTrackerInterval
   402→		fastInterval := 30 * time.Second
   403→
   404→		if normalInterval <= 0 {
   405→			return
   406→		}
   407→
   408→		ticker := time.NewTicker(normalInterval)
   409→		defer ticker.Stop()
   410→		currentInterval := normalInterval
   411→
   412→		for {
   413→			select {
   414→			case <-ctx.Done():
   415→				return
   416→			case <-ticker.C:
   417→				var mem runtime.MemStats
   418→				runtime.ReadMemStats(&mem)
   419→				actualMemoryGB := float64(mem.Alloc) / 1e9
   420→				metricDataGB := ms.SizeInGB()
   421→				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
   422→
   423→				freedExcluded := 0
   424→				freedEmergency := 0
   425→				var err error
   426→
   427→				state.mu.RLock()
   428→				lastRetention := state.lastRetentionTime
   429→				selectorsExcluded := state.selectorsExcluded
   430→				state.mu.RUnlock()
   431→
   432→				if lastRetention != 0 && selectorsExcluded {
   433→					freedExcluded, err = ms.Free(nil, lastRetention)
   434→					if err != nil {
   435→						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
   436→					}
   437→
   438→					if freedExcluded > 0 {
   439→						debug.FreeOSMemory()
   440→						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
   441→					}
   442→				}
   443→
   444→				runtime.ReadMemStats(&mem)
   445→				actualMemoryGB = float64(mem.Alloc) / 1e9
   446→
   447→				if actualMemoryGB > float64(Keys.MemoryCap) {
   448→					cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)
   449→
   450→					// Use progressive time-based Free with increasing threshold
   451→					// instead of ForceFree loop — fewer tree traversals, more effective
   452→					d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
   453→					if parseErr != nil {
   454→						cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
   455→					} else {
   456→						thresholds := []float64{0.75, 0.5, 0.25}
   457→						for _, fraction := range thresholds {
   458→							threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
   459→							freed, freeErr := ms.Free(nil, threshold.Unix())
   460→							if freeErr != nil {
   461→								cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
   462→							}
   463→							freedEmergency += freed
   464→
   465→							bufferPool.Clear()
   466→							runtime.GC()
   467→							runtime.ReadMemStats(&mem)
   468→							actualMemoryGB = float64(mem.Alloc) / 1e9
   469→
   470→							if actualMemoryGB < float64(Keys.MemoryCap) {
   471→								break
   472→							}
   473→						}
   474→					}
   475→
   476→					bufferPool.Clear()
   477→					debug.FreeOSMemory()
   478→
   479→					runtime.ReadMemStats(&mem)
   480→					actualMemoryGB = float64(mem.Alloc) / 1e9
   481→
   482→					if actualMemoryGB >= float64(Keys.MemoryCap) {
   483→						cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
   484→					} else {
   485→						cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
   486→					}
   487→				}
   488→
   489→				// Adaptive ticker: check more frequently when memory is high
   490→				memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
   491→				if memoryRatio > 0.8 && currentInterval != fastInterval {
   492→					ticker.Reset(fastInterval)
   493→					currentInterval = fastInterval
   494→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
   495→				} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
   496→					ticker.Reset(normalInterval)
   497→					currentInterval = normalInterval
   498→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
   499→				}
   500→			}
   501→		}
   502→	})
   503→}
   504→
   505→// Free removes metric data older than the given time while preserving data for active nodes.
   506→//
   507→// This function implements intelligent retention by consulting the NodeProvider (if configured)
   508→// to determine which nodes are currently in use by running jobs. Data for these nodes is
   509→// preserved even if older than the retention time.
   510→//
   511→// Parameters:
   512→//   - ms: The MemoryStore instance
   513→//   - t: Time threshold - buffers with data older than this will be freed
   514→//
   515→// Returns:
   516→//   - Number of buffers freed
   517→//   - Error if NodeProvider query fails
   518→//
   519→// Behavior:
   520→//   - If no NodeProvider is set: frees all buffers older than t
   521→//   - If NodeProvider returns empty map: frees all buffers older than t
   522→//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
   523→func Free(ms *MemoryStore, t time.Time) (int, error) {
   524→	// If no NodeProvider is configured, free all buffers older than t
   525→	if ms.nodeProvider == nil {
   526→		return ms.Free(nil, t.Unix())
   527→	}
   528→
   529→	excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
   530→	if err != nil {
   531→		return 0, err
   532→	}
   533→
   534→	switch lenMap := len(excludeSelectors); lenMap {
   535→
   536→	// If the length of the map returned by GetUsedNodes() is 0,
   537→	// then use default Free method with nil selector
   538→	case 0:
   539→		state.selectorsExcluded = false
   540→		return ms.Free(nil, t.Unix())
   541→
   542→	// Else formulate selectors, exclude those from the map
   543→	// and free the rest of the selectors
   544→	default:
   545→		state.selectorsExcluded = true
   546→		selectors := GetSelectors(ms, excludeSelectors)
   547→		return FreeSelected(ms, selectors, t)
   548→	}
   549→}
   550→
   551→// FreeSelected frees buffers for specific selectors while preserving others.
   552→//
   553→// This function is used when we want to retain some specific nodes beyond the retention time.
   554→// It iterates through the provided selectors and frees their associated buffers.
   555→//
   556→// Parameters:
   557→//   - ms: The MemoryStore instance
   558→//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
   559→//   - t: Time threshold for freeing buffers
   560→//
   561→// Returns the total number of buffers freed and any error encountered.
   562→func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
   563→	freed := 0
   564→
   565→	for _, selector := range selectors {
   566→
   567→		freedBuffers, err := ms.Free(selector, t.Unix())
   568→		if err != nil {
   569→			cclog.Errorf("error while freeing selected buffers: %#v", err)
   570→		}
   571→		freed += freedBuffers
   572→
   573→	}
   574→
   575→	return freed, nil
   576→}
   577→
   578→// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
   579→//
   580→// This function generates a list of selectors whose buffers should be freed by excluding
   581→// selectors that correspond to nodes currently in use by running jobs.
   582→//
   583→// Parameters:
   584→//   - ms: The MemoryStore instance
   585→//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
   586→//
   587→// Returns a list of selectors ([]string paths) that can be safely freed.
   588→//
   589→// Example:
   590→//
   591→//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
   592→//	and excludeSelectors contains {"emmy": ["node001"]},
   593→//	then only [["emmy", "node002"]] is returned.
   594→func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
   595→	allSelectors := ms.GetPaths(2)
   596→
   597→	filteredSelectors := make([][]string, 0, len(allSelectors))
   598→
   599→	for _, path := range allSelectors {
   600→		if len(path) < 2 {
   601→			continue
   602→		}
   603→
   604→		key := path[0]   // The "Key" (Level 1)
   605→		value := path[1] // The "Value" (Level 2)
   606→
   607→		exclude := false
   608→
   609→		// Check if the key exists in our exclusion map
   610→		if excludedValues, exists := excludeSelectors[key]; exists {
   611→			// The key exists, now check if the specific value is in the exclusion list
   612→			if slices.Contains(excludedValues, value) {
   613→				exclude = true
   614→			}
   615→		}
   616→
   617→		if !exclude {
   618→			filteredSelectors = append(filteredSelectors, path)
   619→		}
   620→	}
   621→
   622→	return filteredSelectors
   623→}
   624→
   625→// GetPaths returns a list of lists (paths) to the specified depth.
   626→func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
   627→	var results [][]string
   628→
   629→	// Start recursion. Initial path is empty.
   630→	// We treat Root as depth 0.
   631→	ms.root.collectPaths(0, targetDepth, []string{}, &results)
   632→
   633→	return results
   634→}
   635→
   636→// Write all values in `metrics` to the level specified by `selector` for time `ts`.
   637→// Look at `findLevelOrCreate` for how selectors work.
   638→func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
   639→	var ok bool
   640→	for i, metric := range metrics {
   641→		if metric.MetricConfig.Frequency == 0 {
   642→			metric.MetricConfig, ok = m.Metrics[metric.Name]
   643→			if !ok {
   644→				cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
   645→				metric.MetricConfig.Frequency = 0
   646→			}
   647→			metrics[i] = metric
   648→		}
   649→	}
   650→
   651→	return m.WriteToLevel(&m.root, selector, ts, metrics)
   652→}
   653→
   654→func (m *MemoryStore) GetLevel(selector []string) *Level {
   655→	return m.root.findLevelOrCreate(selector, len(m.Metrics))
   656→}
   657→
   658→// WriteToLevel assumes that `minfo` in `metrics` is filled in
   659→func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
   660→	l = l.findLevelOrCreate(selector, len(m.Metrics))
   661→	l.lock.Lock()
   662→	defer l.lock.Unlock()
   663→
   664→	for _, metric := range metrics {
   665→		if metric.MetricConfig.Frequency == 0 {
   666→			continue
   667→		}
   668→
   669→		b := l.metrics[metric.MetricConfig.offset]
   670→		if b == nil {
   671→			// First write to this metric and level
   672→			b = newBuffer(ts, metric.MetricConfig.Frequency)
   673→			l.metrics[metric.MetricConfig.offset] = b
   674→		}
   675→
   676→		nb, err := b.write(ts, metric.Value)
   677→		if err != nil {
   678→			return err
   679→		}
   680→
   681→		// Last write created a new buffer...
   682→		if b != nb {
   683→			l.metrics[metric.MetricConfig.offset] = nb
   684→		}
   685→	}
   686→	return nil
   687→}
   688→
   689→// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
   690→// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
   691→// The second and third return value are the actual from/to for the data. Those can be different from
   692→// the range asked for if no data was available.
   693→func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
   694→	if from > to {
   695→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
   696→	}
   697→
   698→	minfo, ok := m.Metrics[metric]
   699→	if !ok {
   700→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
   701→	}
   702→
   703→	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
   704→
   705→	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
   706→		cdata, cfrom, cto, err := b.read(from, to, data)
   707→		if err != nil {
   708→			return err
   709→		}
   710→
   711→		if n == 0 {
   712→			from, to = cfrom, cto
   713→		} else if from != cfrom || to != cto || len(data) != len(cdata) {
   714→			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
   715→			if missingfront != 0 {
   716→				return ErrDataDoesNotAlignMissingFront
   717→			}
   718→
   719→			newlen := len(cdata) - missingback
   720→			if newlen < 1 {
   721→				return ErrDataDoesNotAlignMissingBack
   722→			}
   723→			cdata = cdata[0:newlen]
   724→			if len(cdata) != len(data) {
   725→				return ErrDataDoesNotAlignDataLenMismatch
   726→			}
   727→
   728→			from, to = cfrom, cto
   729→		}
   730→
   731→		data = cdata
   732→		n += 1
   733→		return nil
   734→	})
   735→
   736→	if err != nil {
   737→		return nil, 0, 0, 0, err
   738→	} else if n == 0 {
   739→		return nil, 0, 0, 0, ErrNoHostOrMetric
   740→	} else if n > 1 {
   741→		if minfo.Aggregation == AvgAggregation {
   742→			normalize := 1. / schema.Float(n)
   743→			for i := 0; i < len(data); i++ {
   744→				data[i] *= normalize
   745→			}
   746→		} else if minfo.Aggregation != SumAggregation {
   747→			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
   748→		}
   749→	}
   750→
   751→	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
   752→	if err != nil {
   753→		return nil, 0, 0, 0, err
   754→	}
   755→
   756→	return data, from, to, resolution, nil
   757→}
   758→
   759→// Free releases all buffers for the selected level and all its children that
   760→// contain only values older than `t`.
   761→func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
   762→	return m.GetLevel(selector).free(t)
   763→}
   764→
   765→// ForceFree unconditionally removes the oldest buffer from each metric chain.
   766→func (m *MemoryStore) ForceFree() (int, error) {
   767→	return m.GetLevel(nil).forceFree()
   768→}
   769→
   770→func (m *MemoryStore) FreeAll() error {
   771→	m.root.lock.Lock()
   772→	defer m.root.lock.Unlock()
   773→
   774→	for k := range m.root.children {
   775→		delete(m.root.children, k)
   776→	}
   777→
   778→	return nil
   779→}
   780→
   781→func (m *MemoryStore) SizeInBytes() int64 {
   782→	return m.root.sizeInBytes()
   783→}
   784→
   785→func (m *MemoryStore) SizeInGB() float64 {
   786→	return float64(m.root.sizeInBytes()) / 1e9
   787→}
   788→
   789→// ListChildren , given a selector, returns a list of all children of the level
   790→// selected.
   791→func (m *MemoryStore) ListChildren(selector []string) []string {
   792→	lvl := &m.root
   793→	for lvl != nil && len(selector) != 0 {
   794→		lvl.lock.RLock()
   795→		next := lvl.children[selector[0]]
   796→		lvl.lock.RUnlock()
   797→		lvl = next
   798→		selector = selector[1:]
   799→	}
   800→
   801→	if lvl == nil {
   802→		return nil
   803→	}
   804→
   805→	lvl.lock.RLock()
   806→	defer lvl.lock.RUnlock()
   807→
   808→	children := make([]string, 0, len(lvl.children))
   809→	for child := range lvl.children {
   810→		children = append(children, child)
   811→	}
   812→
   813→	return children
   814→}
   815→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


Now I have all three files. Let me analyze the initialization and shutdown sequences in detail.

## Analysis Report

I've read all three files completely. Here's my detailed analysis:

### 1. **Init() Parameters and Side Effects** (metricstore.go lines 125-194)

Init() expects:
- `rawConfig`: JSON configuration (may be nil)
- `metrics`: Map of metric names to MetricConfig
- `wg`: WaitGroup pointer

Side effects:
- Decodes and validates configuration
- Calculates NumWorkers from config or defaults to `min(runtime.NumCPU()/2+1, 10)`
- Calls `InitMetrics(metrics)` to initialize the singleton
- Loads checkpoints from disk via `FromCheckpointFiles()`
- **Creates a context and cancellation function** at line 171: `ctx, shutdown := context.WithCancel(context.Background())`
- Starts 5 background goroutines by calling:
  - `Retention(wg, ctx)`
  - `Checkpointing(wg, ctx)`
  - `CleanUp(wg, ctx)`
  - `WALStaging(wg, ctx)`
  - `MemoryUsageTracker(wg, ctx)`
- **Stores the cancellation function in global `shutdownFunc`** (lines 183-185)
- If NATS subscriptions are configured, starts `ReceiveNats()` goroutine (lines 187-193)

**Each background goroutine calls `wg.Go()` to register itself with the WaitGroup.**

### 2. **Shutdown() Sequence** (metricstore.go lines 274-321)

The shutdown sequence in metricstore:
1. **Cancels all background goroutines** by calling `shutdownFunc()` (line 280)
2. For WAL format checkpoints:
   - Sets `walShuttingDown` flag to prevent panics
   - Sleeps 100ms to allow in-flight operations to complete
   - Closes WAL channels
   - Waits for WAL staging goroutines to drain
3. **Writes a final checkpoint** to disk (lines 299-315)

**Critical issue:** The shutdown function does NOT wait for the background goroutines to actually exit via `wg.Wait()`.

### 3. **Server.Shutdown() Sequence** (server.go lines 125-142)

```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
    defer cancel()

    nc := nats.GetClient()
    if nc != nil {
        nc.Close()  // ← Close NATS FIRST
    }

    if err := s.server.Shutdown(shutdownCtx); err != nil {  // ← Shutdown HTTP SECOND
        cclog.Errorf("Server shutdown error: %v", err)
    }

    metricstore.Shutdown()  // ← Shutdown METRICSTORE LAST
}
```

**Order:** NATS → HTTP Server → Metricstore

### 4. **WaitGroup Usage in cc-metric-store**

Looking at main.go (runServer function, lines 49-123):
- A WaitGroup `wg` is created at line 50
- Passed to `metricstore.Init()` at line 63
- Background goroutines register via `wg.Go()` inside metricstore
- HTTP server is started via `wg.Go()` at line 86
- Signal handler registered via `wg.Add(1)` at line 93
- **At line 112: `wg.Wait()` is called** in a goroutine that closes errChan
- However, **`wg.Wait()` is never explicitly awaited** in runServer itself

The shutdown flow:
1. Signal handler receives signal → calls `srv.Shutdown(ctx)` (line 105)
2. `srv.Shutdown()` calls `metricstore.Shutdown()` (server.go line 141)
3. `metricstore.Shutdown()` cancels context (wakes up background workers)
4. HTTP server's `Serve()` returns when context is cancelled
5. `wg.Wait()` completes (all goroutines done)
6. `errChan` is closed
7. `runServer()` returns

### 5. **GOGC Settings**

In main.go lines 65-68:
```go
if os.Getenv(envGOGC) == "" {
    debug.SetGCPercent(15)
}
```

**Issue:** This is set AFTER `metricstore.Init()` is called at line 63. The GC percent is only set to 15% after the metricstore has already loaded checkpoints and started workers.

### 6. **gops Initialization Order**

In main.go run() function:
1. Line 126-132: Parse flags (gops flag parsed here)
2. Line 139: Initialize logging
3. **Lines 141-145: gops agent starts BEFORE config is loaded** ← **PROBLEM**
4. Line 147: `ccconf.Init(flagConfigFile)` loads config
5. Line 149: Main config is retrieved

The gops agent is started at line 142 **before** the configuration is loaded from disk at line 147. However, line 141 checks `config.Keys.Debug.EnableGops` which hasn't been loaded yet, so this condition will be false on first check.

---

## Summary of Issues Found

1. **WaitGroup not explicitly waited in runServer context**: The `wg.Wait()` happens in a background goroutine, not blocking the main flow. However, the error channel mechanism ensures proper sequencing.

2. **GOGC set after metricstore.Init()**: The GC percent (15%) is set after checkpoint loading and worker startup, not before.

3. **gops initialization timing**: The gops agent starts before the config file is loaded, though it uses a flag check that won't be true at that point.

4. **metricstore.Shutdown() doesn't wait for workers**: The function cancels the context but doesn't explicitly wait for all background goroutines to exit via the WaitGroup. The final checkpoint write happens immediately after cancellation, which could race with in-flight operations.

agentId: a1983d38da575f6da (use SendMessage with to: 'a1983d38da575f6da' to continue this agent)
<usage>total_tokens: 32789
tool_uses: 3
duration_ms: 17549</usage>

Prompt:
Read the following files completely and return their full content:

1. /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go — focus on the Init() and Shutdown() functions and any global vars they use
2. /Users/jan/prg/ClusterCockpit/cc-metric-store/cmd/cc-metric-store/main.go — already known, but confirm the runServer() and run() functions
3. /Users/jan/prg/ClusterCockpit/cc-metric-store/cmd/cc-metric-store/server.go — focus on Shutdown() sequence

I need to compare exactly:
- What Init() expects as parameters and what side effects it has
- What Shutdown() does step by step
- Whether the cc-metric-store's server.go Shutdown() calls metricstore.Shutdown() at the right time relative to NATS and HTTP shutdown
- Whether the WaitGroup passed to Init() is waited on somewhere in cc-metric-store
- Whether GOGC / GC settings are still relevant
- Whether gops initialization order is correct (it happens before config is loaded in run())


Let me read the key files directly to verify these findings.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-metric-store.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package main provides the entry point for the ClusterCockpit metric store server.
     7→// This file contains HTTP server setup, routing configuration.
     8→package main
     9→
    10→import (
    11→	"context"
    12→	"crypto/tls"
    13→	"fmt"
    14→	"net"
    15→	"net/http"
    16→	"time"
    17→
    18→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    19→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    20→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    21→	"github.com/ClusterCockpit/cc-lib/v2/runtime"
    22→	"github.com/ClusterCockpit/cc-metric-store/internal/api"
    23→	"github.com/ClusterCockpit/cc-metric-store/internal/config"
    24→	httpSwagger "github.com/swaggo/http-swagger/v2"
    25→)
    26→
    27→// Server encapsulates the HTTP server state and dependencies
    28→type Server struct {
    29→	router *http.ServeMux
    30→	server *http.Server
    31→}
    32→
    33→// NewServer creates and initializes a new Server instance
    34→func NewServer(version, commit, buildDate string) (*Server, error) {
    35→	s := &Server{
    36→		router: http.NewServeMux(),
    37→	}
    38→
    39→	if err := s.init(); err != nil {
    40→		return nil, err
    41→	}
    42→
    43→	return s, nil
    44→}
    45→
    46→func (s *Server) init() error {
    47→	api.MountRoutes(s.router)
    48→
    49→	if flagDev {
    50→		cclog.Print("Enable Swagger UI!")
    51→		s.router.HandleFunc("GET /swagger/", httpSwagger.Handler(
    52→			httpSwagger.URL("http://"+config.Keys.Address+"/swagger/doc.json")))
    53→	}
    54→
    55→	return nil
    56→}
    57→
    58→// Server timeout defaults (in seconds)
    59→const (
    60→	defaultReadTimeout  = 30
    61→	defaultWriteTimeout = 30
    62→)
    63→
    64→func (s *Server) Start(ctx context.Context) error {
    65→	// Use configurable timeouts with defaults
    66→	readTimeout := time.Duration(defaultReadTimeout) * time.Second
    67→	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second
    68→
    69→	s.server = &http.Server{
    70→		ReadTimeout:  readTimeout,
    71→		WriteTimeout: writeTimeout,
    72→		Handler:      s.router,
    73→		Addr:         config.Keys.Address,
    74→	}
    75→
    76→	// Start http or https server
    77→	listener, err := net.Listen("tcp", config.Keys.Address)
    78→	if err != nil {
    79→		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Address, err)
    80→	}
    81→
    82→	if config.Keys.CertFile != "" && config.Keys.KeyFile != "" {
    83→		cert, err := tls.LoadX509KeyPair(
    84→			config.Keys.CertFile, config.Keys.KeyFile)
    85→		if err != nil {
    86→			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
    87→		}
    88→		listener = tls.NewListener(listener, &tls.Config{
    89→			Certificates: []tls.Certificate{cert},
    90→			CipherSuites: []uint16{
    91→				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
    92→				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
    93→			},
    94→			MinVersion:               tls.VersionTLS12,
    95→			PreferServerCipherSuites: true,
    96→		})
    97→		cclog.Infof("HTTPS server listening at %s...", config.Keys.Address)
    98→	} else {
    99→		cclog.Infof("HTTP server listening at %s...", config.Keys.Address)
   100→	}
   101→
   102→	// Because this program will want to bind to a privileged port (like 80), the listener must
   103→	// be established first, then the user can be changed, and after that,
   104→	// the actual http server can be started.
   105→	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
   106→		return fmt.Errorf("dropping privileges: %w", err)
   107→	}
   108→
   109→	// Handle context cancellation for graceful shutdown
   110→	go func() {
   111→		<-ctx.Done()
   112→		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
   113→		defer cancel()
   114→		if err := s.server.Shutdown(shutdownCtx); err != nil {
   115→			cclog.Errorf("Server shutdown error: %v", err)
   116→		}
   117→	}()
   118→
   119→	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
   120→		return fmt.Errorf("server failed: %w", err)
   121→	}
   122→	return nil
   123→}
   124→
   125→func (s *Server) Shutdown(ctx context.Context) {
   126→	// Create a shutdown context with timeout
   127→	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
   128→	defer cancel()
   129→
   130→	nc := nats.GetClient()
   131→	if nc != nil {
   132→		nc.Close()
   133→	}
   134→
   135→	// First shut down the server gracefully (waiting for all ongoing requests)
   136→	if err := s.server.Shutdown(shutdownCtx); err != nil {
   137→		cclog.Errorf("Server shutdown error: %v", err)
   138→	}
   139→
   140→	// Archive all the metric store data
   141→	metricstore.Shutdown()
   142→}
   143→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-metric-store.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package main provides the entry point for the ClusterCockpit metric store server.
// This file contains HTTP server setup, routing configuration.
package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/runtime"
	"github.com/ClusterCockpit/cc-metric-store/internal/api"
	"github.com/ClusterCockpit/cc-metric-store/internal/config"
	httpSwagger "github.com/swaggo/http-swagger/v2"
)

// Server encapsulates the HTTP server state and dependencies
type Server struct {
	router *http.ServeMux
	server *http.Server
}

// NewServer creates and initializes a new Server instance
func NewServer(version, commit, buildDate string) (*Server, error) {
	s := &Server{
		router: http.NewServeMux(),
	}

	if err := s.init(); err != nil {
		return nil, err
	}

	return s, nil
}

func (s *Server) init() error {
	api.MountRoutes(s.router)

	if flagDev {
		cclog.Print("Enable Swagger UI!")
		s.router.HandleFunc("GET /swagger/", httpSwagger.Handler(
			httpSwagger.URL("http://"+config.Keys.Address+"/swagger/doc.json")))
	}

	return nil
}

// Server timeout defaults (in seconds)
const (
	defaultReadTimeout  = 30
	defaultWriteTimeout = 30
)

func (s *Server) Start(ctx context.Context) error {
	// Use configurable timeouts with defaults
	readTimeout := time.Duration(defaultReadTimeout) * time.Second
	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second

	s.server = &http.Server{
		ReadTimeout:  readTimeout,
		WriteTimeout: writeTimeout,
		Handler:      s.router,
		Addr:         config.Keys.Address,
	}

	// Start http or https server
	listener, err := net.Listen("tcp", config.Keys.Address)
	if err != nil {
		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Address, err)
	}

	if config.Keys.CertFile != "" && config.Keys.KeyFile != "" {
		cert, err := tls.LoadX509KeyPair(
			config.Keys.CertFile, config.Keys.KeyFile)
		if err != nil {
			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
		}
		listener = tls.NewListener(listener, &tls.Config{
			Certificates: []tls.Certificate{cert},
			CipherSuites: []uint16{
				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			},
			MinVersion:               tls.VersionTLS12,
			PreferServerCipherSuites: true,
		})
		cclog.Infof("HTTPS server listening at %s...", config.Keys.Address)
	} else {
		cclog.Infof("HTTP server listening at %s...", config.Keys.Address)
	}

	// Because this program will want to bind to a privileged port (like 80), the listener must
	// be established first, then the user can be changed, and after that,
	// the actual http server can be started.
	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
		return fmt.Errorf("dropping privileges: %w", err)
	}

	// Handle context cancellation for graceful shutdown
	go func() {
		<-ctx.Done()
		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		if err := s.server.Shutdown(shutdownCtx); err != nil {
			cclog.Errorf("Server shutdown error: %v", err)
		}
	}()

	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
		return fmt.Errorf("server failed: %w", err)
	}
	return nil
}

func (s *Server) Shutdown(ctx context.Context) {
	// Create a shutdown context with timeout
	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	// First shut down the server gracefully (waiting for all ongoing requests)
	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Archive all the metric store data
	metricstore.Shutdown()
}


   125→func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
   126→	startupTime := time.Now()
   127→
   128→	if rawConfig != nil {
   129→		config.Validate(configSchema, rawConfig)
   130→		dec := json.NewDecoder(bytes.NewReader(rawConfig))
   131→		dec.DisallowUnknownFields()
   132→		if err := dec.Decode(&Keys); err != nil {
   133→			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
   134→		}
   135→	}
   136→
   137→	// Set NumWorkers from config or use default
   138→	if Keys.NumWorkers <= 0 {
   139→		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
   140→	}
   141→	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
   142→
   143→	// Pass the config.MetricStoreKeys
   144→	InitMetrics(metrics)
   145→
   146→	ms := GetMemoryStore()
   147→
   148→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   149→	if err != nil {
   150→		cclog.Fatal(err)
   151→	}
   152→
   153→	restoreFrom := startupTime.Add(-d)
   154→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   155→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   156→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   157→	if err != nil {
   158→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   159→	} else {
   160→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   161→	}
   162→
   163→	// Try to use less memory by forcing a GC run here and then
   164→	// lowering the target percentage. The default of 100 means
   165→	// that only once the ratio of new allocations execeds the
   166→	// previously active heap, a GC is triggered.
   167→	// Forcing a GC here will set the "previously active heap"
   168→	// to a minumum.
   169→	// runtime.GC()
   170→
   171→	ctx, shutdown := context.WithCancel(context.Background())
   172→
   173→	Retention(wg, ctx)
   174→	Checkpointing(wg, ctx)
   175→	CleanUp(wg, ctx)
   176→	WALStaging(wg, ctx)
   177→	MemoryUsageTracker(wg, ctx)
   178→
   179→	// Note: Signal handling has been removed from this function.
   180→	// The caller is responsible for handling shutdown signals and calling
   181→	// the shutdown() function when appropriate.
   182→	// Store the shutdown function for later use by Shutdown()
   183→	shutdownFuncMu.Lock()
   184→	shutdownFunc = shutdown
   185→	shutdownFuncMu.Unlock()
   186→
   187→	if Keys.Subscriptions != nil {
   188→		wg.Go(func() {
   189→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   190→				cclog.Fatal(err)
   191→			}
   192→		})
   193→	}
   194→}
   195→
   196→// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
   197→//
   198→// This function must be called before GetMemoryStore() and can only be called once due to
   199→// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
   200→//
   201→// Parameters:
   202→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
   203→//
   204→// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
   205→//
   206→// After this call, the global msInstance is ready for use via GetMemoryStore().
   207→func InitMetrics(metrics map[string]MetricConfig) {
   208→	singleton.Do(func() {
   209→		offset := 0
   210→		for key, cfg := range metrics {
   211→			if cfg.Frequency == 0 {
   212→				panic("[METRICSTORE]> invalid frequency")
   213→			}
   214→
   215→			metrics[key] = MetricConfig{
   216→				Frequency:   cfg.Frequency,
   217→				Aggregation: cfg.Aggregation,
   218→				offset:      offset,
   219→			}
   220→			offset += 1
   221→		}
   222→
   223→		msInstance = &MemoryStore{
   224→			root: Level{
   225→				metrics:  make([]*buffer, len(metrics)),
   226→				children: make(map[string]*Level),
   227→			},
   228→			Metrics: metrics,
   229→		}
   230→	})
   231→}
   232→
   233→// GetMemoryStore returns the singleton MemoryStore instance.
   234→//
   235→// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
   236→//
   237→// This function is safe for concurrent use after initialization.
   238→func GetMemoryStore() *MemoryStore {
   239→	if msInstance == nil {
   240→		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
   241→	}
   242→
   243→	return msInstance
   244→}
   245→
   246→func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
   247→	if metric, ok := ms.Metrics[metricName]; ok {
   248→		return metric.Frequency, nil
   249→	}
   250→	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
   251→}
   252→
   253→// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
   254→// This must be called during initialization to provide job state information
   255→// for selective buffer retention during Free operations.
   256→// If not set, the Free function will fall back to freeing all buffers.
   257→func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
   258→	ms.nodeProvider = provider
   259→}
   260→
   261→// Shutdown performs a graceful shutdown of the metric store.
   262→//
   263→// This function cancels all background goroutines started by Init() and writes
   264→// a final checkpoint to disk before returning. It should be called when the
   265→// application receives a termination signal.
   266→//
   267→// The function will:
   268→//  1. Cancel the context to stop all background workers
   269→//  2. Close the WAL messages channel if using WAL format
   270→//  3. Write a final checkpoint to preserve in-memory data
   271→//  4. Log any errors encountered during shutdown
   272→//
   273→// Note: This function blocks until the final checkpoint is written.
   274→func Shutdown() {
   275→	totalStart := time.Now()
   276→
   277→	shutdownFuncMu.Lock()
   278→	defer shutdownFuncMu.Unlock()
   279→	if shutdownFunc != nil {
   280→		shutdownFunc()
   281→	}
   282→	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))
   283→
   284→	if Keys.Checkpoints.FileFormat == "wal" {
   285→		// Signal producers to stop sending before closing channels,
   286→		// preventing send-on-closed-channel panics from in-flight NATS workers.
   287→		walShuttingDown.Store(true)
   288→		// Brief grace period for in-flight DecodeLine calls to complete.
   289→		time.Sleep(100 * time.Millisecond)
   290→
   291→		for _, ch := range walShardChs {
   292→			close(ch)
   293→		}
   294→		drainStart := time.Now()
   295→		WaitForWALStagingDrain()
   296→		cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
   297→	}
   298→
   299→	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
   300→	checkpointStart := time.Now()
   301→	var files int
   302→	var err error
   303→
   304→	ms := GetMemoryStore()
   305→
   306→	lastCheckpointMu.Lock()
   307→	from := lastCheckpoint
   308→	lastCheckpointMu.Unlock()
   309→
   310→	if Keys.Checkpoints.FileFormat == "wal" {
   311→		// WAL files are deleted per-host inside ToCheckpointWAL workers.
   312→		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   313→	} else {
   314→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   315→	}
   316→
   317→	if err != nil {
   318→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
   319→	}
   320→	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
   321→}
   322→
   323→// Retention starts a background goroutine that periodically frees old metric data.
   324→//

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
	startupTime := time.Now()

	if rawConfig != nil {
		config.Validate(configSchema, rawConfig)
		dec := json.NewDecoder(bytes.NewReader(rawConfig))
		dec.DisallowUnknownFields()
		if err := dec.Decode(&Keys); err != nil {
			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
		}
	}

	// Set NumWorkers from config or use default
	if Keys.NumWorkers <= 0 {
		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
	}
	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)

	// Pass the config.MetricStoreKeys
	InitMetrics(metrics)

	ms := GetMemoryStore()

	d, err := time.ParseDuration(Keys.RetentionInMemory)
	if err != nil {
		cclog.Fatal(err)
	}

	restoreFrom := startupTime.Add(-d)
	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
	if err != nil {
		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
	} else {
		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
	}

	// Try to use less memory by forcing a GC run here and then
	// lowering the target percentage. The default of 100 means
	// that only once the ratio of new allocations execeds the
	// previously active heap, a GC is triggered.
	// Forcing a GC here will set the "previously active heap"
	// to a minumum.
	// runtime.GC()

	ctx, shutdown := context.WithCancel(context.Background())

	Retention(wg, ctx)
	Checkpointing(wg, ctx)
	CleanUp(wg, ctx)
	WALStaging(wg, ctx)
	MemoryUsageTracker(wg, ctx)

	// Note: Signal handling has been removed from this function.
	// The caller is responsible for handling shutdown signals and calling
	// the shutdown() function when appropriate.
	// Store the shutdown function for later use by Shutdown()
	shutdownFuncMu.Lock()
	shutdownFunc = shutdown
	shutdownFuncMu.Unlock()

	if Keys.Subscriptions != nil {
		wg.Go(func() {
			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
				cclog.Fatal(err)
			}
		})
	}
}

// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
//
// This function must be called before GetMemoryStore() and can only be called once due to
// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
//
// Parameters:
//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
//
// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
//
// After this call, the global msInstance is ready for use via GetMemoryStore().
func InitMetrics(metrics map[string]MetricConfig) {
	singleton.Do(func() {
		offset := 0
		for key, cfg := range metrics {
			if cfg.Frequency == 0 {
				panic("[METRICSTORE]> invalid frequency")
			}

			metrics[key] = MetricConfig{
				Frequency:   cfg.Frequency,
				Aggregation: cfg.Aggregation,
				offset:      offset,
			}
			offset += 1
		}

		msInstance = &MemoryStore{
			root: Level{
				metrics:  make([]*buffer, len(metrics)),
				children: make(map[string]*Level),
			},
			Metrics: metrics,
		}
	})
}

// GetMemoryStore returns the singleton MemoryStore instance.
//
// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
//
// This function is safe for concurrent use after initialization.
func GetMemoryStore() *MemoryStore {
	if msInstance == nil {
		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
	}

	return msInstance
}

func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
	if metric, ok := ms.Metrics[metricName]; ok {
		return metric.Frequency, nil
	}
	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
}

// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
// This must be called during initialization to provide job state information
// for selective buffer retention during Free operations.
// If not set, the Free function will fall back to freeing all buffers.
func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
	ms.nodeProvider = provider
}

// Shutdown performs a graceful shutdown of the metric store.
//
// This function cancels all background goroutines started by Init() and writes
// a final checkpoint to disk before returning. It should be called when the
// application receives a termination signal.
//
// The function will:
//  1. Cancel the context to stop all background workers
//  2. Close the WAL messages channel if using WAL format
//  3. Write a final checkpoint to preserve in-memory data
//  4. Log any errors encountered during shutdown
//
// Note: This function blocks until the final checkpoint is written.
func Shutdown() {
	totalStart := time.Now()

	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}
	cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))

	if Keys.Checkpoints.FileFormat == "wal" {
		// Signal producers to stop sending before closing channels,
		// preventing send-on-closed-channel panics from in-flight NATS workers.
		walShuttingDown.Store(true)
		// Brief grace period for in-flight DecodeLine calls to complete.
		time.Sleep(100 * time.Millisecond)

		for _, ch := range walShardChs {
			close(ch)
		}
		drainStart := time.Now()
		WaitForWALStagingDrain()
		cclog.Infof("[METRICSTORE]> WAL staging goroutines exited (%v)", time.Since(drainStart))
	}

	cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
	checkpointStart := time.Now()
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		// WAL files are deleted per-host inside ToCheckpointWAL workers.
		files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written in %v, total shutdown: %v)", files, time.Since(checkpointStart), time.Since(totalStart))
}

// Retention starts a background goroutine that periodically frees old metric data.
//


Now I have a complete picture. Let me update the plan.


Write

# Plan: Fix startup/shutdown sequence vs metricstore

## Context

After comparing cc-metric-store's `main.go`/`server.go` against what `cc-backend/pkg/metricstore`'s
`Init()` and `Shutdown()` actually expect, two real bugs were found in the startup order.
The shutdown sequence itself is correct (NATS → HTTP → metricstore.Shutdown()).

## Bugs found

### Bug 1 — `EnableGops` config key is never effective (`cmd/cc-metric-store/main.go`)

**Code (lines 141-145):**
```go
if flagGops || config.Keys.Debug.EnableGops {   // ← checked BEFORE config loaded
    agent.Listen(...)
}

ccconf.Init(flagConfigFile)   // line 147
config.Init(cfg)              // line 154 — only here is config.Keys populated
```

`config.Keys.Debug.EnableGops` is always `false` (zero value) at line 141 because
`config.Init()` hasn't been called yet. The `"debug": { "gops": true }` option in
`config.json` silently has no effect.

**Fix:** Move the gops block to after `config.Init(cfg)`.

### Bug 2 — GOGC set after metricstore.Init() (`cmd/cc-metric-store/main.go`, `runServer()`)

**Code (lines 63-68):**
```go
metricstore.Init(mscfg, config.GetMetrics(), &wg)   // loads checkpoints, starts workers

if os.Getenv(envGOGC) == "" {
    debug.SetGCPercent(15)
}
```

The GC target is set to 15% *after* checkpoint loading, which is the largest allocation
event at startup. The metricstore's own Init() has a commented-out `runtime.GC()` call
(line 169) that was intended to establish a good GC baseline after loading — that only
works well if the GC percent is already low at that point.

**Fix:** Move `debug.SetGCPercent(15)` to before `metricstore.Init()` in `runServer()`.

## Shutdown sequence — confirmed correct

The order in `server.go Shutdown()` is:
1. Close NATS client → stops new inbound metric data
2. `s.server.Shutdown()` → drains in-flight HTTP requests
3. `metricstore.Shutdown()` → cancels workers, drains WAL staging, writes final checkpoint

This matches what `metricstore.Shutdown()` requires (no open write paths when checkpoint is written).

## Files to change

- `cmd/cc-metric-store/main.go`
  - Move gops block (lines 141-145) to after `config.Init(cfg)` (currently line 154)
  - Move `debug.SetGCPercent(15)` block to before `metricstore.Init()` in `runServer()`

## Verification

```bash
go build ./...
go test -v ./...
```

Manually: run with `"debug": { "gops": true }` in config.json and confirm gops agent starts.


The file /Users/jan/.claude/plans/stateful-frolicking-jellyfish.md has been updated successfully.

# Plan: Fix startup/shutdown sequence vs metricstore

## Context

After comparing cc-metric-store's `main.go`/`server.go` against what `cc-backend/pkg/metricstore`'s
`Init()` and `Shutdown()` actually expect, two real bugs were found in the startup order.
The shutdown sequence itself is correct (NATS → HTTP → metricstore.Shutdown()).

## Bugs found

### Bug 1 — `EnableGops` config key is never effective (`cmd/cc-metric-store/main.go`)

**Code (lines 141-145):**
```go
if flagGops || config.Keys.Debug.EnableGops {   // ← checked BEFORE config loaded
    agent.Listen(...)
}

ccconf.Init(flagConfigFile)   // line 147
config.Init(cfg)              // line 154 — only here is config.Keys populated
```

`config.Keys.Debug.EnableGops` is always `false` (zero value) at line 141 because
`config.Init()` hasn't been called yet. The `"debug": { "gops": true }` option in
`config.json` silently has no effect.

**Fix:** Move the gops block to after `config.Init(cfg)`.

### Bug 2 — GOGC set after metricstore.Init() (`cmd/cc-metric-store/main.go`, `runServer()`)

**Code (lines 63-68):**
```go
metricstore.Init(mscfg, config.GetMetrics(), &wg)   // loads checkpoints, starts workers

if os.Getenv(envGOGC) == "" {
    debug.SetGCPercent(15)
}
```

The GC target is set to 15% *after* checkpoint loading, which is the largest allocation
event at startup. The metricstore's own Init() has a commented-out `runtime.GC()` call
(line 169) that was intended to establish a good GC baseline after loading — that only
works well if the GC percent is already low at that point.

**Fix:** Move `debug.SetGCPercent(15)` to before `metricstore.Init()` in `runServer()`.

## Shutdown sequence — confirmed correct

The order in `server.go Shutdown()` is:
1. Close NATS client → stops new inbound metric data
2. `s.server.Shutdown()` → drains in-flight HTTP requests
3. `metricstore.Shutdown()` → cancels workers, drains WAL staging, writes final checkpoint

This matches what `metricstore.Shutdown()` requires (no open write paths when checkpoint is written).

## Files to change

- `cmd/cc-metric-store/main.go`
  - Move gops block (lines 141-145) to after `config.Init(cfg)` (currently line 154)
  - Move `debug.SetGCPercent(15)` block to before `metricstore.Init()` in `runServer()`

## Verification

```bash
go build ./...
go test -v ./...
```

Manually: run with `"debug": { "gops": true }` in config.json and confirm gops agent starts.

-# Plan: Sync metric-store config and docs with cc-backend

+# Plan: Fix startup/shutdown sequence vs metricstore

-The `MetricStoreConfig` struct in `cc-backend/pkg/metricstore/config.go` has evolved. The

-cc-metric-store's config template and documentation are out of sync. Specifically:

+After comparing cc-metric-store's `main.go`/`server.go` against what `cc-backend/pkg/metricstore`'s

+`Init()` and `Shutdown()` actually expect, two real bugs were found in the startup order.

+The shutdown sequence itself is correct (NATS → HTTP → metricstore.Shutdown()).

-1. `checkpoints.interval` was moved to a top-level `checkpoint-interval` field

-2. `cleanup.interval` was removed (now derived from `retention-in-memory`)

-3. A new `checkpoints.file-format` field (`"json"` or `"wal"`) was added; `"wal"` is now the default

-4. New optional fields: `num-workers`, `checkpoints.max-wal-size`

-5. README incorrectly says `memory-cap` is in MB (it's GB)

-6. README incorrectly labels `"json"` as the default checkpoint format (it's `"wal"`)

+### Bug 1 — `EnableGops` config key is never effective (`cmd/cc-metric-store/main.go`)

-    "interval": "12h",       ← INVALID field (moved to top-level)

-    "interval": "48h",       ← INVALID field (derived from retention-in-memory)

+if flagGops || config.Keys.Debug.EnableGops {   // ← checked BEFORE config loaded

+ccconf.Init(flagConfigFile)   // line 147

+config.Init(cfg)              // line 154 — only here is config.Keys populated

-- Add top-level `checkpoint-interval: "12h"`

+`config.Keys.Debug.EnableGops` is always `false` (zero value) at line 141 because

+`config.Init()` hasn't been called yet. The `"debug": { "gops": true }` option in

+**Fix:** Move the gops block to after `config.Init(cfg)`.

-Two description bugs (the example JSON block is already correct):

+### Bug 2 — GOGC set after metricstore.Init() (`cmd/cc-metric-store/main.go`, `runServer()`)

-- Line ~180: `memory-cap` description says "MB" → fix to "GB"

-- Line ~177: `checkpoints.file-format` says `"json"` (default) → fix to `"wal"` (default)

-- "Checkpoint formats" section: `**\`"json"\` (default)**` header → remove `(default)`, and `**\`"wal"\`**` → add `(default)` to WAL header

+metricstore.Init(mscfg, config.GetMetrics(), &wg)   // loads checkpoints, starts workers

-Update `metric-store` config description to reflect:

-- `checkpoint-interval` as top-level field (no longer inside `checkpoints`)

-- Removal of `cleanup.interval` (derived)

-- New fields: `num-workers`, `checkpoints.file-format`, `checkpoints.max-wal-size`

+The GC target is set to 15% *after* checkpoint loading, which is the largest allocation

+event at startup. The metricstore's own Init() has a commented-out `runtime.GC()` call

+(line 169) that was intended to establish a good GC baseline after loading — that only

+works well if the GC percent is already low at that point.

+**Fix:** Move `debug.SetGCPercent(15)` to before `metricstore.Init()` in `runServer()`.

+## Shutdown sequence — confirmed correct

-- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go` — authoritative struct

-- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/configSchema.go` — authoritative JSON schema

+The order in `server.go Shutdown()` is:

+1. Close NATS client → stops new inbound metric data

+2. `s.server.Shutdown()` → drains in-flight HTTP requests

+3. `metricstore.Shutdown()` → cancels workers, drains WAL staging, writes final checkpoint

+This matches what `metricstore.Shutdown()` requires (no open write paths when checkpoint is written).

+  - Move gops block (lines 141-145) to after `config.Init(cfg)` (currently line 154)

+  - Move `debug.SetGCPercent(15)` block to before `metricstore.Init()` in `runServer()`

-- `go build ./...` in cc-metric-store (config is only read at runtime, but build confirms no breakage)

-- Manually inspect that `configs/config.json` passes the schema from `configSchema.go` (no unknown fields)

+Manually: run with `"debug": { "gops": true }` in config.json and confirm gops agent starts.

# Plan: Sync metric-store config and docs with cc-backend

## Context

The `MetricStoreConfig` struct in `cc-backend/pkg/metricstore/config.go` has evolved. The
cc-metric-store's config template and documentation are out of sync. Specifically:

1. `checkpoints.interval` was moved to a top-level `checkpoint-interval` field
2. `cleanup.interval` was removed (now derived from `retention-in-memory`)
3. A new `checkpoints.file-format` field (`"json"` or `"wal"`) was added; `"wal"` is now the default
4. New optional fields: `num-workers`, `checkpoints.max-wal-size`
5. README incorrectly says `memory-cap` is in MB (it's GB)
6. README incorrectly labels `"json"` as the default checkpoint format (it's `"wal"`)

## Files to change

### 1. `configs/config.json`

Current broken `metric-store` section:
```json
"metric-store": {
  "checkpoints": {
    "interval": "12h",       ← INVALID field (moved to top-level)
    "directory": "./var/checkpoints"
  },
  "memory-cap": 100,
  "retention-in-memory": "48h",
  "cleanup": {
    "mode": "archive",
    "interval": "48h",       ← INVALID field (derived from retention-in-memory)
    "directory": "./var/archive"
  },
  "nats-subscriptions": [...]
}
```

Fix:
- Remove `checkpoints.interval`
- Remove `cleanup.interval`
- Add `checkpoints.file-format: "wal"`
- Add top-level `checkpoint-interval: "12h"`

### 2. `README.md`

Two description bugs (the example JSON block is already correct):

- Line ~180: `memory-cap` description says "MB" → fix to "GB"
- Line ~177: `checkpoints.file-format` says `"json"` (default) → fix to `"wal"` (default)
- "Checkpoint formats" section: `**\`"json"\` (default)**` header → remove `(default)`, and `**\`"wal"\`**` → add `(default)` to WAL header

### 3. `CLAUDE.md`

Update `metric-store` config description to reflect:
- `checkpoint-interval` as top-level field (no longer inside `checkpoints`)
- WAL format as the default
- Removal of `cleanup.interval` (derived)
- New fields: `num-workers`, `checkpoints.file-format`, `checkpoints.max-wal-size`

## Critical files

- `/Users/jan/prg/ClusterCockpit/cc-metric-store/configs/config.json`
- `/Users/jan/prg/ClusterCockpit/cc-metric-store/README.md`
- `/Users/jan/prg/ClusterCockpit/cc-metric-store/CLAUDE.md`

Reference (read-only):
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go` — authoritative struct
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/configSchema.go` — authoritative JSON schema

## Verification

- `go build ./...` in cc-metric-store (config is only read at runtime, but build confirms no breakage)
- Manually inspect that `configs/config.json` passes the schema from `configSchema.go` (no unknown fields)
- Review README render for correctness


ExitPlanMode

# Plan: Fix startup/shutdown sequence vs metricstore

## Context

After comparing cc-metric-store's `main.go`/`server.go` against what `cc-backend/pkg/metricstore`'s
`Init()` and `Shutdown()` actually expect, two real bugs were found in the startup order.
The shutdown sequence itself is correct (NATS → HTTP → metricstore.Shutdown()).

## Bugs found

### Bug 1 — `EnableGops` config key is never effective (`cmd/cc-metric-store/main.go`)

**Code (lines 141-145):**
```go
if flagGops || config.Keys.Debug.EnableGops {   // ← checked BEFORE config loaded
    agent.Listen(...)
}

ccconf.Init(flagConfigFile)   // line 147
config.Init(cfg)              // line 154 — only here is config.Keys populated
```

`config.Keys.Debug.EnableGops` is always `false` (zero value) at line 141 because
`config.Init()` hasn't been called yet. The `"debug": { "gops": true }` option in
`config.json` silently has no effect.

**Fix:** Move the gops block to after `config.Init(cfg)`.

### Bug 2 — GOGC set after metricstore.Init() (`cmd/cc-metric-store/main.go`, `runServer()`)

**Code (lines 63-68):**
```go
metricstore.Init(mscfg, config.GetMetrics(), &wg)   // loads checkpoints, starts workers

if os.Getenv(envGOGC) == "" {
    debug.SetGCPercent(15)
}
```

The GC target is set to 15% *after* checkpoint loading, which is the largest allocation
event at startup. The metricstore's own Init() has a commented-out `runtime.GC()` call
(line 169) that was intended to establish a good GC baseline after loading — that only
works well if the GC percent is already low at that point.

**Fix:** Move `debug.SetGCPercent(15)` to before `metricstore.Init()` in `runServer()`.

## Shutdown sequence — confirmed correct

The order in `server.go Shutdown()` is:
1. Close NATS client → stops new inbound metric data
2. `s.server.Shutdown()` → drains in-flight HTTP requests
3. `metricstore.Shutdown()` → cancels workers, drains WAL staging, writes final checkpoint

This matches what `metricstore.Shutdown()` requires (no open write paths when checkpoint is written).

## Files to change

- `cmd/cc-metric-store/main.go`
  - Move gops block (lines 141-145) to after `config.Init(cfg)` (currently line 154)
  - Move `debug.SetGCPercent(15)` block to before `metricstore.Init()` in `runServer()`

## Verification

```bash
go build ./...
go test -v ./...
```

Manually: run with `"debug": { "gops": true }` in config.json and confirm gops agent starts.


User has approved your plan. You can now start coding. Start with updating your todo list if applicable

Your plan has been saved to: /Users/jan/.claude/plans/stateful-frolicking-jellyfish.md
You can refer back to it if needed during implementation.

## Approved Plan:
# Plan: Fix startup/shutdown sequence vs metricstore

## Context

After comparing cc-metric-store's `main.go`/`server.go` against what `cc-backend/pkg/metricstore`'s
`Init()` and `Shutdown()` actually expect, two real bugs were found in the startup order.
The shutdown sequence itself is correct (NATS → HTTP → metricstore.Shutdown()).

## Bugs found

### Bug 1 — `EnableGops` config key is never effective (`cmd/cc-metric-store/main.go`)

**Code (lines 141-145):**
```go
if flagGops || config.Keys.Debug.EnableGops {   // ← checked BEFORE config loaded
    agent.Listen(...)
}

ccconf.Init(flagConfigFile)   // line 147
config.Init(cfg)              // line 154 — only here is config.Keys populated
```

`config.Keys.Debug.EnableGops` is always `false` (zero value) at line 141 because
`config.Init()` hasn't been called yet. The `"debug": { "gops": true }` option in
`config.json` silently has no effect.

**Fix:** Move the gops block to after `config.Init(cfg)`.

### Bug 2 — GOGC set after metricstore.Init() (`cmd/cc-metric-store/main.go`, `runServer()`)

**Code (lines 63-68):**
```go
metricstore.Init(mscfg, config.GetMetrics(), &wg)   // loads checkpoints, starts workers

if os.Getenv(envGOGC) == "" {
    debug.SetGCPercent(15)
}
```

The GC target is set to 15% *after* checkpoint loading, which is the largest allocation
event at startup. The metricstore's own Init() has a commented-out `runtime.GC()` call
(line 169) that was intended to establish a good GC baseline after loading — that only
works well if the GC percent is already low at that point.

**Fix:** Move `debug.SetGCPercent(15)` to before `metricstore.Init()` in `runServer()`.

## Shutdown sequence — confirmed correct

The order in `server.go Shutdown()` is:
1. Close NATS client → stops new inbound metric data
2. `s.server.Shutdown()` → drains in-flight HTTP requests
3. `metricstore.Shutdown()` → cancels workers, drains WAL staging, writes final checkpoint

This matches what `metricstore.Shutdown()` requires (no open write paths when checkpoint is written).

## Files to change

- `cmd/cc-metric-store/main.go`
  - Move gops block (lines 141-145) to after `config.Init(cfg)` (currently line 154)
  - Move `debug.SetGCPercent(15)` block to before `metricstore.Init()` in `runServer()`

## Verification

```bash
go build ./...
go test -v ./...
```

Manually: run with `"debug": { "gops": true }` in config.json and confirm gops agent starts.


Two fixes in `main.go`:


Edit

	cclog.Init(flagLogLevel, flagLogDateTime)

	if flagGops || config.Keys.Debug.EnableGops {
		if err := agent.Listen(agent.Options{}); err != nil {
			return fmt.Errorf("starting gops agent: %w", err)
		}
	}

	ccconf.Init(flagConfigFile)

	cfg := ccconf.GetPackageConfig("main")
	if cfg == nil {
		return fmt.Errorf("main configuration must be present")
	}

	config.Init(cfg)

	cclog.Init(flagLogLevel, flagLogDateTime)

	ccconf.Init(flagConfigFile)

	cfg := ccconf.GetPackageConfig("main")
	if cfg == nil {
		return fmt.Errorf("main configuration must be present")
	}

	config.Init(cfg)

	if flagGops || config.Keys.Debug.EnableGops {
		if err := agent.Listen(agent.Options{}); err != nil {
			return fmt.Errorf("starting gops agent: %w", err)
		}
	}


The file /Users/jan/prg/ClusterCockpit/cc-metric-store/cmd/cc-metric-store/main.go has been updated successfully.

	cclog.Init(flagLogLevel, flagLogDateTime)

	ccconf.Init(flagConfigFile)

	cfg := ccconf.GetPackageConfig("main")
	if cfg == nil {
		return fmt.Errorf("main configuration must be present")
	}

	config.Init(cfg)

	if flagGops || config.Keys.Debug.EnableGops {
		if err := agent.Listen(agent.Options{}); err != nil {
			return fmt.Errorf("starting gops agent: %w", err)
		}
	}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-metric-store.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"runtime/debug"
	"sync"
	"syscall"

	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/runtime"
	"github.com/ClusterCockpit/cc-metric-store/internal/api"
	"github.com/ClusterCockpit/cc-metric-store/internal/config"
	"github.com/google/gops/agent"
)

const (
	envGOGC = "GOGC"
)

var (
	date    string
	commit  string
	version string
)

var (
	flagGops, flagVersion, flagDev, flagLogDateTime bool
	flagConfigFile, flagLogLevel                    string
)

func printVersion() {
	fmt.Printf("Version:\t%s\n", version)
	fmt.Printf("Git hash:\t%s\n", commit)
	fmt.Printf("Build time:\t%s\n", date)
}

func runServer(ctx context.Context) error {
	var wg sync.WaitGroup

	mscfg := ccconf.GetPackageConfig("metrics")
	if mscfg == nil {
		return fmt.Errorf("missing metrics configuration")
	}
	config.InitMetrics(mscfg)

	mscfg = ccconf.GetPackageConfig("metric-store")
	if mscfg == nil {
		return fmt.Errorf("missing metricstore configuration")
	}

	metricstore.Init(mscfg, config.GetMetrics(), &wg)

	// Set GC percent if not configured
	if os.Getenv(envGOGC) == "" {
		debug.SetGCPercent(15)
	}

	if config.Keys.BackendURL != "" {
		ms := metricstore.GetMemoryStore()
		ms.SetNodeProvider(api.NewBackendNodeProvider(config.Keys.BackendURL))
		cclog.Infof("Node provider configured with backend URL: %s", config.Keys.BackendURL)
	}

	// Initialize HTTP server
	srv, err := NewServer(version, commit, date)
	if err != nil {
		return fmt.Errorf("creating server: %w", err)
	}

	// Channel to collect errors from server
	errChan := make(chan error, 1)

	// Start HTTP server
	wg.Go(func() {
		if err := srv.Start(ctx); err != nil {
			errChan <- err
		}
	})

	// Handle shutdown signals
	wg.Add(1)
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		defer wg.Done()
		select {
		case <-sigs:
			cclog.Info("Shutdown signal received")
		case <-ctx.Done():
		}

		runtime.SystemdNotify(false, "Shutting down ...")
		srv.Shutdown(ctx)
	}()

	runtime.SystemdNotify(true, "running")

	// Wait for completion or errors
	go func() {
		wg.Wait()
		close(errChan)
	}()

	// Wait for either server startup error or shutdown completion
	if err := <-errChan; err != nil {
		return err
	}

	cclog.Print("Graceful shutdown completed!")
	return nil
}

func run() error {
	flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)")
	flag.BoolVar(&flagDev, "dev", false, "Enable development component: Swagger UI")
	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info, warn (default), err, crit]`")
	flag.Parse()

	if flagVersion {
		printVersion()
		return nil
	}

	cclog.Init(flagLogLevel, flagLogDateTime)

	if flagGops || config.Keys.Debug.EnableGops {
		if err := agent.Listen(agent.Options{}); err != nil {
			return fmt.Errorf("starting gops agent: %w", err)
		}
	}

	ccconf.Init(flagConfigFile)

	cfg := ccconf.GetPackageConfig("main")
	if cfg == nil {
		return fmt.Errorf("main configuration must be present")
	}

	config.Init(cfg)

	natsConfig := ccconf.GetPackageConfig("nats")
	if err := nats.Init(natsConfig); err != nil {
		cclog.Warnf("initializing (optional) nats client: %s", err.Error())
	}
	nats.Connect()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	return runServer(ctx)
}

func main() {
	if err := run(); err != nil {
		cclog.Error(err.Error())
		os.Exit(1)
	}
}

-  if flagGops || config.Keys.Debug.EnableGops {

-    if err := agent.Listen(agent.Options{}); err != nil {

-      return fmt.Errorf("starting gops agent: %w", err)

+  if flagGops || config.Keys.Debug.EnableGops {

+    if err := agent.Listen(agent.Options{}); err != nil {

+      return fmt.Errorf("starting gops agent: %w", err)


	metricstore.Init(mscfg, config.GetMetrics(), &wg)

	// Set GC percent if not configured
	if os.Getenv(envGOGC) == "" {
		debug.SetGCPercent(15)
	}

	if config.Keys.BackendURL != "" {

	// Set GC percent before loading checkpoints so the GC baseline is established
	// with a low target from the start of the largest allocation event.
	if os.Getenv(envGOGC) == "" {
		debug.SetGCPercent(15)
	}

	metricstore.Init(mscfg, config.GetMetrics(), &wg)

	if config.Keys.BackendURL != "" {


	// Set GC percent before loading checkpoints so the GC baseline is established
	// with a low target from the start of the largest allocation event.
	if os.Getenv(envGOGC) == "" {
		debug.SetGCPercent(15)
	}

	metricstore.Init(mscfg, config.GetMetrics(), &wg)

	if config.Keys.BackendURL != "" {

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-metric-store.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"runtime/debug"
	"sync"
	"syscall"

	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/runtime"
	"github.com/ClusterCockpit/cc-metric-store/internal/api"
	"github.com/ClusterCockpit/cc-metric-store/internal/config"
	"github.com/google/gops/agent"
)

const (
	envGOGC = "GOGC"
)

var (
	date    string
	commit  string
	version string
)

var (
	flagGops, flagVersion, flagDev, flagLogDateTime bool
	flagConfigFile, flagLogLevel                    string
)

func printVersion() {
	fmt.Printf("Version:\t%s\n", version)
	fmt.Printf("Git hash:\t%s\n", commit)
	fmt.Printf("Build time:\t%s\n", date)
}

func runServer(ctx context.Context) error {
	var wg sync.WaitGroup

	mscfg := ccconf.GetPackageConfig("metrics")
	if mscfg == nil {
		return fmt.Errorf("missing metrics configuration")
	}
	config.InitMetrics(mscfg)

	mscfg = ccconf.GetPackageConfig("metric-store")
	if mscfg == nil {
		return fmt.Errorf("missing metricstore configuration")
	}

	metricstore.Init(mscfg, config.GetMetrics(), &wg)

	// Set GC percent if not configured
	if os.Getenv(envGOGC) == "" {
		debug.SetGCPercent(15)
	}

	if config.Keys.BackendURL != "" {
		ms := metricstore.GetMemoryStore()
		ms.SetNodeProvider(api.NewBackendNodeProvider(config.Keys.BackendURL))
		cclog.Infof("Node provider configured with backend URL: %s", config.Keys.BackendURL)
	}

	// Initialize HTTP server
	srv, err := NewServer(version, commit, date)
	if err != nil {
		return fmt.Errorf("creating server: %w", err)
	}

	// Channel to collect errors from server
	errChan := make(chan error, 1)

	// Start HTTP server
	wg.Go(func() {
		if err := srv.Start(ctx); err != nil {
			errChan <- err
		}
	})

	// Handle shutdown signals
	wg.Add(1)
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		defer wg.Done()
		select {
		case <-sigs:
			cclog.Info("Shutdown signal received")
		case <-ctx.Done():
		}

		runtime.SystemdNotify(false, "Shutting down ...")
		srv.Shutdown(ctx)
	}()

	runtime.SystemdNotify(true, "running")

	// Wait for completion or errors
	go func() {
		wg.Wait()
		close(errChan)
	}()

	// Wait for either server startup error or shutdown completion
	if err := <-errChan; err != nil {
		return err
	}

	cclog.Print("Graceful shutdown completed!")
	return nil
}

func run() error {
	flag.BoolVar(&flagGops, "gops", false, "Listen via github.com/google/gops/agent (for debugging)")
	flag.BoolVar(&flagDev, "dev", false, "Enable development component: Swagger UI")
	flag.BoolVar(&flagVersion, "version", false, "Show version information and exit")
	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info, warn (default), err, crit]`")
	flag.Parse()

	if flagVersion {
		printVersion()
		return nil
	}

	cclog.Init(flagLogLevel, flagLogDateTime)

	ccconf.Init(flagConfigFile)

	cfg := ccconf.GetPackageConfig("main")
	if cfg == nil {
		return fmt.Errorf("main configuration must be present")
	}

	config.Init(cfg)

	if flagGops || config.Keys.Debug.EnableGops {
		if err := agent.Listen(agent.Options{}); err != nil {
			return fmt.Errorf("starting gops agent: %w", err)
		}
	}

	natsConfig := ccconf.GetPackageConfig("nats")
	if err := nats.Init(natsConfig); err != nil {
		cclog.Warnf("initializing (optional) nats client: %s", err.Error())
	}
	nats.Connect()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	return runServer(ctx)
}

func main() {
	if err := run(); err != nil {
		cclog.Error(err.Error())
		os.Exit(1)
	}
}

-  metricstore.Init(mscfg, config.GetMetrics(), &wg)

+  // Set GC percent before loading checkpoints so the GC baseline is established

+  // with a low target from the start of the largest allocation event.

+  metricstore.Init(mscfg, config.GetMetrics(), &wg)


Bash

Build and vet cc-metric-store


(Bash completed with no output)


Clean. Here's what was changed:

**Bug 1 — gops never activated from config file**: Moved the gops initialization block to *after* `config.Init(cfg)`. Previously `config.Keys.Debug.EnableGops` was always `false` (zero value) because it was checked before the config file was parsed. The `-gops` CLI flag worked, but `"debug": {"gops": true}` in `config.json` silently had no effect.

**Bug 2 — GOGC set after checkpoint loading**: Moved `debug.SetGCPercent(15)` to *before* `metricstore.Init()`. The GC target is now 15% during the checkpoint load (the largest allocation event at startup), which is exactly when it matters most for memory control.

**Shutdown sequence** — confirmed correct as-is: NATS close → HTTP drain → `metricstore.Shutdown()` (WAL drain + final checkpoint). No changes needed there.


Are the command line options complete compared to @../cc-backend/ ?