   150→// WALStaging starts N sharded background goroutines that receive WALMessage items
   151→// and append binary WAL records to per-host current.wal files.
   152→// Each shard owns a subset of hosts (determined by hash of cluster+node),
   153→// parallelizing serialization and I/O while keeping per-host writes serial.
   154→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   155→	if Keys.Checkpoints.FileFormat == "json" {
   156→		return
   157→	}
   158→
   159→	walNumShards = max(Keys.NumWorkers, 1)
   160→	chBufSize := max(65536/walNumShards, 1024)
   161→
   162→	walShardChs = make([]chan *WALMessage, walNumShards)
   163→	walShardRotateChs = make([]chan walRotateReq, walNumShards)
   164→
   165→	for i := range walNumShards {
   166→		walShardChs[i] = make(chan *WALMessage, chBufSize)
   167→		walShardRotateChs[i] = make(chan walRotateReq, 64)
   168→	}
   169→
   170→	for i := range walNumShards {
   171→		msgCh := walShardChs[i]
   172→		rotateCh := walShardRotateChs[i]
   173→
   174→		wg.Go(func() {
   175→			hostFiles := make(map[string]*walFileState)
   176→
   177→			defer func() {
   178→				for _, ws := range hostFiles {
   179→					if ws.f != nil {
   180→						ws.w.Flush()
   181→						ws.f.Close()
   182→					}
   183→				}
   184→			}()
   185→
   186→			getOrOpenWAL := func(hostDir string) *walFileState {
   187→				ws, ok := hostFiles[hostDir]
   188→				if ok {
   189→					return ws
   190→				}
   191→
   192→				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   193→					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   194→					return nil
   195→				}
   196→
   197→				walPath := path.Join(hostDir, "current.wal")
   198→				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   199→				if err != nil {
   200→					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   201→					return nil
   202→				}
   203→
   204→				w := bufio.NewWriter(f)
   205→
   206→				// Write file header magic if file is new (empty).
   207→				info, err := f.Stat()
   208→				if err == nil && info.Size() == 0 {
   209→					var hdr [4]byte
   210→					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   211→					if _, err := w.Write(hdr[:]); err != nil {
   212→						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   213→						f.Close()
   214→						return nil
   215→					}
   216→				}
   217→
   218→				ws = &walFileState{f: f, w: w}
   219→				hostFiles[hostDir] = ws
   220→				return ws
   221→			}
   222→
   223→			processMsg := func(msg *WALMessage) {
   224→				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   225→				ws := getOrOpenWAL(hostDir)
   226→				if ws == nil {
   227→					return
   228→				}
   229→				if err := writeWALRecordDirect(ws.w, msg); err != nil {
   230→					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   231→				}
   232→				ws.dirty = true
   233→			}
   234→
   235→			processRotate := func(req walRotateReq) {
   236→				ws, ok := hostFiles[req.hostDir]
   237→				if ok && ws.f != nil {
   238→					ws.w.Flush()
   239→					ws.f.Close()
   240→					walPath := path.Join(req.hostDir, "current.wal")
   241→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   242→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   243→					}
   244→					delete(hostFiles, req.hostDir)
   245→				}
   246→				close(req.done)
   247→			}
   248→
   249→			flushDirty := func() {
   250→				for _, ws := range hostFiles {
   251→					if ws.dirty {
   252→						ws.w.Flush()
   253→						ws.dirty = false
   254→					}
   255→				}
   256→			}
   257→
   258→			drain := func() {
   259→				for {
   260→					select {
   261→					case msg, ok := <-msgCh:
   262→						if !ok {
   263→							return
   264→						}
   265→						processMsg(msg)
   266→					case req := <-rotateCh:
   267→						processRotate(req)
   268→					default:
   269→						flushDirty()
   270→						return
   271→					}
   272→				}
   273→			}
   274→
   275→			ticker := time.NewTicker(walFlushInterval)
   276→			defer ticker.Stop()
   277→
   278→			// drainBatch processes up to 4096 pending messages without blocking.
   279→			// Returns false if the channel was closed.
   280→			drainBatch := func() bool {
   281→				for range 4096 {
   282→					select {
   283→					case msg, ok := <-msgCh:
   284→						if !ok {
   285→							flushDirty()
   286→							return false
   287→						}
   288→						processMsg(msg)
   289→					case req := <-rotateCh:
   290→						processRotate(req)
   291→					default:
   292→						return true
   293→					}
   294→				}
   295→				return true
   296→			}
   297→
   298→			for {
   299→				select {
   300→				case <-ctx.Done():
   301→					drain()
   302→					return
   303→				case msg, ok := <-msgCh:
   304→					if !ok {
   305→						return
   306→					}
   307→					processMsg(msg)
   308→					if !drainBatch() {
   309→						return
   310→					}
   311→					// No flush here — timer handles periodic flushing.
   312→				case <-ticker.C:
   313→					flushDirty()
   314→				case req := <-rotateCh:
   315→					processRotate(req)
   316→				}
   317→			}
   318→		})
   319→	}
   320→}
   321→
   322→// RotateWALFiles sends rotation requests for the given host directories
   323→// and blocks until all rotations complete. Each request is routed to the
   324→// shard that owns the host directory.
   325→func RotateWALFiles(hostDirs []string) {
   326→	if walShardRotateChs == nil {
   327→		return
   328→	}
   329→	dones := make([]chan struct{}, len(hostDirs))
   330→	for i, dir := range hostDirs {
   331→		dones[i] = make(chan struct{})
   332→		// Extract cluster/node from hostDir to find the right shard.
   333→		// hostDir = rootDir/cluster/node
   334→		shard := walShardIndexFromDir(dir)
   335→		walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: dones[i]}
   336→	}
   337→	for _, done := range dones {
   338→		<-done
   339→	}
   340→}
   341→
   342→// walShardIndexFromDir extracts cluster and node from a host directory path
   343→// and returns the shard index. The path format is: rootDir/cluster/node
   344→func walShardIndexFromDir(hostDir string) int {
   345→	// hostDir = .../cluster/node — extract last two path components
   346→	node := path.Base(hostDir)
   347→	cluster := path.Base(path.Dir(hostDir))
   348→	return walShardIndex(cluster, node)
   349→}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.
// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return
	}

	walNumShards = max(Keys.NumWorkers, 1)
	chBufSize := max(65536/walNumShards, 1024)

	walShardChs = make([]chan *WALMessage, walNumShards)
	walShardRotateChs = make([]chan walRotateReq, walNumShards)

	for i := range walNumShards {
		walShardChs[i] = make(chan *WALMessage, chBufSize)
		walShardRotateChs[i] = make(chan walRotateReq, 64)
	}

	for i := range walNumShards {
		msgCh := walShardChs[i]
		rotateCh := walShardRotateChs[i]

		wg.Go(func() {
			hostFiles := make(map[string]*walFileState)

			defer func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
						ws.f.Close()
					}
				}
			}()

			getOrOpenWAL := func(hostDir string) *walFileState {
				ws, ok := hostFiles[hostDir]
				if ok {
					return ws
				}

				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
					return nil
				}

				walPath := path.Join(hostDir, "current.wal")
				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
					return nil
				}

				w := bufio.NewWriter(f)

				// Write file header magic if file is new (empty).
				info, err := f.Stat()
				if err == nil && info.Size() == 0 {
					var hdr [4]byte
					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
					if _, err := w.Write(hdr[:]); err != nil {
						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
						f.Close()
						return nil
					}
				}

				ws = &walFileState{f: f, w: w}
				hostFiles[hostDir] = ws
				return ws
			}

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}
				if err := writeWALRecordDirect(ws.w, msg); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
				ws.dirty = true
			}

			processRotate := func(req walRotateReq) {
				ws, ok := hostFiles[req.hostDir]
				if ok && ws.f != nil {
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(req.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, req.hostDir)
				}
				close(req.done)
			}

			flushDirty := func() {
				for _, ws := range hostFiles {
					if ws.dirty {
						ws.w.Flush()
						ws.dirty = false
					}
				}
			}

			drain := func() {
				for {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							return
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						flushDirty()
						return
					}
				}
			}

			ticker := time.NewTicker(walFlushInterval)
			defer ticker.Stop()

			// drainBatch processes up to 4096 pending messages without blocking.
			// Returns false if the channel was closed.
			drainBatch := func() bool {
				for range 4096 {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							flushDirty()
							return false
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						return true
					}
				}
				return true
			}

			for {
				select {
				case <-ctx.Done():
					drain()
					return
				case msg, ok := <-msgCh:
					if !ok {
						return
					}
					processMsg(msg)
					if !drainBatch() {
						return
					}
					// No flush here — timer handles periodic flushing.
				case <-ticker.C:
					flushDirty()
				case req := <-rotateCh:
					processRotate(req)
				}
			}
		})
	}
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete. Each request is routed to the
// shard that owns the host directory.
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil {
		return
	}
	dones := make([]chan struct{}, len(hostDirs))
	for i, dir := range hostDirs {
		dones[i] = make(chan struct{})
		// Extract cluster/node from hostDir to find the right shard.
		// hostDir = rootDir/cluster/node
		shard := walShardIndexFromDir(dir)
		walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: dones[i]}
	}
	for _, done := range dones {
		<-done
	}
}

// walShardIndexFromDir extracts cluster and node from a host directory path
// and returns the shard index. The path format is: rootDir/cluster/node
func walShardIndexFromDir(hostDir string) int {
	// hostDir = .../cluster/node — extract last two path components
	node := path.Base(hostDir)
	cluster := path.Base(path.Dir(hostDir))
	return walShardIndex(cluster, node)
}


    80→
    81→// lastCheckpoint tracks the timestamp of the last checkpoint creation.
    82→var (
    83→	lastCheckpoint   time.Time
    84→	lastCheckpointMu sync.Mutex
    85→)
    86→
    87→// Checkpointing starts a background worker that periodically saves metric data to disk.
    88→//
    89→// Checkpoints are written every 12 hours (hardcoded).
    90→//
    91→// Format behaviour:
    92→//   - "json": Periodic checkpointing every checkpointInterval
    93→//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
    94→func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
    95→	lastCheckpointMu.Lock()
    96→	lastCheckpoint = time.Now()
    97→	lastCheckpointMu.Unlock()
    98→
    99→	ms := GetMemoryStore()
   100→
   101→	wg.Go(func() {
   102→
   103→		d := 12 * time.Hour // default checkpoint interval
   104→		if Keys.CheckpointInterval != "" {
   105→			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
   106→			if err != nil {
   107→				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
   108→			} else {
   109→				d = parsed
   110→			}
   111→		}
   112→
   113→		ticker := time.NewTicker(d)
   114→		defer ticker.Stop()
   115→
   116→		for {
   117→			select {
   118→			case <-ctx.Done():
   119→				return
   120→			case <-ticker.C:
   121→				lastCheckpointMu.Lock()
   122→				from := lastCheckpoint
   123→				lastCheckpointMu.Unlock()
   124→
   125→				now := time.Now()
   126→				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
   127→
   128→				if Keys.Checkpoints.FileFormat == "wal" {
   129→					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   130→					if err != nil {
   131→						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
   132→					} else {
   133→						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
   134→						lastCheckpointMu.Lock()
   135→						lastCheckpoint = now
   136→						lastCheckpointMu.Unlock()
   137→						// Rotate WAL files for successfully checkpointed hosts.
   138→						RotateWALFiles(hostDirs)
   139→					}
   140→				} else {
   141→					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   142→					if err != nil {
   143→						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
   144→					} else {
   145→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
   146→						lastCheckpointMu.Lock()
   147→						lastCheckpoint = now
   148→						lastCheckpointMu.Unlock()
   149→					}
   150→				}
   151→			}
   152→		}
   153→	})
   154→}
   155→
   156→// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
   157→//
   158→// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
   159→// This method manually constructs JSON to avoid allocations and interface conversions.

// lastCheckpoint tracks the timestamp of the last checkpoint creation.
var (
	lastCheckpoint   time.Time
	lastCheckpointMu sync.Mutex
)

// Checkpointing starts a background worker that periodically saves metric data to disk.
//
// Checkpoints are written every 12 hours (hardcoded).
//
// Format behaviour:
//   - "json": Periodic checkpointing every checkpointInterval
//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
	lastCheckpointMu.Lock()
	lastCheckpoint = time.Now()
	lastCheckpointMu.Unlock()

	ms := GetMemoryStore()

	wg.Go(func() {

		d := 12 * time.Hour // default checkpoint interval
		if Keys.CheckpointInterval != "" {
			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
			if err != nil {
				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
			} else {
				d = parsed
			}
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				lastCheckpointMu.Lock()
				from := lastCheckpoint
				lastCheckpointMu.Unlock()

				now := time.Now()
				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))

				if Keys.Checkpoints.FileFormat == "wal" {
					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
					} else {
						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
						// Rotate WAL files for successfully checkpointed hosts.
						RotateWALFiles(hostDirs)
					}
				} else {
					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
					}
				}
			}
		}
	})
}

// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
//
// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
// This method manually constructs JSON to avoid allocations and interface conversions.


Now let me check the WALStaging goroutines — when `close(ch)` is called in `Shutdown()`, the WALStaging goroutines need to drain and exit. But `Shutdown()` doesn't wait for them.


   160→	}
   161→
   162→	// Try to use less memory by forcing a GC run here and then
   163→	// lowering the target percentage. The default of 100 means
   164→	// that only once the ratio of new allocations execeds the
   165→	// previously active heap, a GC is triggered.
   166→	// Forcing a GC here will set the "previously active heap"
   167→	// to a minumum.
   168→	// runtime.GC()
   169→
   170→	ctx, shutdown := context.WithCancel(context.Background())
   171→
   172→	Retention(wg, ctx)
   173→	Checkpointing(wg, ctx)
   174→	CleanUp(wg, ctx)
   175→	WALStaging(wg, ctx)
   176→	MemoryUsageTracker(wg, ctx)
   177→
   178→	// Note: Signal handling has been removed from this function.
   179→	// The caller is responsible for handling shutdown signals and calling
   180→	// the shutdown() function when appropriate.
   181→	// Store the shutdown function for later use by Shutdown()
   182→	shutdownFuncMu.Lock()
   183→	shutdownFunc = shutdown
   184→	shutdownFuncMu.Unlock()
   185→
   186→	if Keys.Subscriptions != nil {
   187→		wg.Go(func() {
   188→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   189→				cclog.Fatal(err)
   190→			}
   191→		})
   192→	}
   193→}
   194→
   195→// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
   196→//
   197→// This function must be called before GetMemoryStore() and can only be called once due to
   198→// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
   199→//
   200→// Parameters:
   201→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
   202→//
   203→// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
   204→//
   205→// After this call, the global msInstance is ready for use via GetMemoryStore().
   206→func InitMetrics(metrics map[string]MetricConfig) {
   207→	singleton.Do(func() {
   208→		offset := 0
   209→		for key, cfg := range metrics {
   210→			if cfg.Frequency == 0 {
   211→				panic("[METRICSTORE]> invalid frequency")
   212→			}
   213→
   214→			metrics[key] = MetricConfig{
   215→				Frequency:   cfg.Frequency,
   216→				Aggregation: cfg.Aggregation,
   217→				offset:      offset,
   218→			}
   219→			offset += 1
   220→		}
   221→
   222→		msInstance = &MemoryStore{
   223→			root: Level{
   224→				metrics:  make([]*buffer, len(metrics)),
   225→				children: make(map[string]*Level),
   226→			},
   227→			Metrics: metrics,
   228→		}
   229→	})
   230→}
   231→
   232→// GetMemoryStore returns the singleton MemoryStore instance.
   233→//
   234→// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
   235→//
   236→// This function is safe for concurrent use after initialization.
   237→func GetMemoryStore() *MemoryStore {
   238→	if msInstance == nil {
   239→		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
   240→	}
   241→
   242→	return msInstance
   243→}
   244→
   245→func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
   246→	if metric, ok := ms.Metrics[metricName]; ok {
   247→		return metric.Frequency, nil
   248→	}
   249→	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
   250→}
   251→
   252→// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
   253→// This must be called during initialization to provide job state information
   254→// for selective buffer retention during Free operations.
   255→// If not set, the Free function will fall back to freeing all buffers.
   256→func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
   257→	ms.nodeProvider = provider
   258→}
   259→
   260→// Shutdown performs a graceful shutdown of the metric store.
   261→//
   262→// This function cancels all background goroutines started by Init() and writes
   263→// a final checkpoint to disk before returning. It should be called when the
   264→// application receives a termination signal.
   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→

	}

	// Try to use less memory by forcing a GC run here and then
	// lowering the target percentage. The default of 100 means
	// that only once the ratio of new allocations execeds the
	// previously active heap, a GC is triggered.
	// Forcing a GC here will set the "previously active heap"
	// to a minumum.
	// runtime.GC()

	ctx, shutdown := context.WithCancel(context.Background())

	Retention(wg, ctx)
	Checkpointing(wg, ctx)
	CleanUp(wg, ctx)
	WALStaging(wg, ctx)
	MemoryUsageTracker(wg, ctx)

	// Note: Signal handling has been removed from this function.
	// The caller is responsible for handling shutdown signals and calling
	// the shutdown() function when appropriate.
	// Store the shutdown function for later use by Shutdown()
	shutdownFuncMu.Lock()
	shutdownFunc = shutdown
	shutdownFuncMu.Unlock()

	if Keys.Subscriptions != nil {
		wg.Go(func() {
			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
				cclog.Fatal(err)
			}
		})
	}
}

// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
//
// This function must be called before GetMemoryStore() and can only be called once due to
// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
//
// Parameters:
//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
//
// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
//
// After this call, the global msInstance is ready for use via GetMemoryStore().
func InitMetrics(metrics map[string]MetricConfig) {
	singleton.Do(func() {
		offset := 0
		for key, cfg := range metrics {
			if cfg.Frequency == 0 {
				panic("[METRICSTORE]> invalid frequency")
			}

			metrics[key] = MetricConfig{
				Frequency:   cfg.Frequency,
				Aggregation: cfg.Aggregation,
				offset:      offset,
			}
			offset += 1
		}

		msInstance = &MemoryStore{
			root: Level{
				metrics:  make([]*buffer, len(metrics)),
				children: make(map[string]*Level),
			},
			Metrics: metrics,
		}
	})
}

// GetMemoryStore returns the singleton MemoryStore instance.
//
// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
//
// This function is safe for concurrent use after initialization.
func GetMemoryStore() *MemoryStore {
	if msInstance == nil {
		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
	}

	return msInstance
}

func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
	if metric, ok := ms.Metrics[metricName]; ok {
		return metric.Frequency, nil
	}
	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
}

// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
// This must be called during initialization to provide job state information
// for selective buffer retention during Free operations.
// If not set, the Free function will fall back to freeing all buffers.
func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
	ms.nodeProvider = provider
}

// Shutdown performs a graceful shutdown of the metric store.
//
// This function cancels all background goroutines started by Init() and writes
// a final checkpoint to disk before returning. It should be called when the
// application receives a termination signal.
//
// The function will:
//  1. Cancel the context to stop all background workers
//  2. Close the WAL messages channel if using WAL format
//  3. Write a final checkpoint to preserve in-memory data
//  4. Log any errors encountered during shutdown
//
// Note: This function blocks until the final checkpoint is written.
func Shutdown() {
	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}

	if Keys.Checkpoints.FileFormat == "wal" {
		for _, ch := range walShardChs {
			close(ch)
		}
	}

	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
	var files int
	var err error


   260→
   261→// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
   262→// Skips metrics that are already archived. Returns nil if no unarchived data exists.
   263→func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
   264→	l.lock.RLock()
   265→	defer l.lock.RUnlock()
   266→
   267→	retval := &CheckpointFile{
   268→		From:     from,
   269→		To:       to,
   270→		Metrics:  make(map[string]*CheckpointMetrics),
   271→		Children: make(map[string]*CheckpointFile),
   272→	}
   273→
   274→	for metric, minfo := range m.Metrics {
   275→		b := l.metrics[minfo.offset]
   276→		if b == nil {
   277→			continue
   278→		}
   279→
   280→		allArchived := true
   281→		b.iterFromTo(from, to, func(b *buffer) error {
   282→			if !b.archived {
   283→				allArchived = false
   284→				return fmt.Errorf("stop") // Early termination signal
   285→			}
   286→			return nil
   287→		})
   288→
   289→		if allArchived {
   290→			continue
   291→		}
   292→
   293→		data := make([]schema.Float, (to-from)/b.frequency+1)
   294→		data, start, end, err := b.read(from, to, data)
   295→		if err != nil {
   296→			return nil, err
   297→		}
   298→
   299→		for i := int((end - start) / b.frequency); i < len(data); i++ {
   300→			data[i] = schema.NaN
   301→		}
   302→
   303→		retval.Metrics[metric] = &CheckpointMetrics{
   304→			Frequency: b.frequency,
   305→			Start:     start,
   306→			Data:      data,
   307→		}
   308→	}
   309→
   310→	for name, child := range l.children {
   311→		val, err := child.toCheckpointFile(from, to, m)
   312→		if err != nil {
   313→			return nil, err
   314→		}
   315→
   316→		if val != nil {
   317→			retval.Children[name] = val
   318→		}
   319→	}
   320→
   321→	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
   322→		return nil, nil
   323→	}
   324→
   325→	return retval, nil
   326→}
   327→
   328→// toCheckpoint writes a Level's data to a JSON checkpoint file.
   329→// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.
   330→func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
   331→	cf, err := l.toCheckpointFile(from, to, m)
   332→	if err != nil {
   333→		return err
   334→	}
   335→
   336→	if cf == nil {
   337→		return ErrNoNewArchiveData
   338→	}
   339→

// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
// Skips metrics that are already archived. Returns nil if no unarchived data exists.
func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	retval := &CheckpointFile{
		From:     from,
		To:       to,
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	for metric, minfo := range m.Metrics {
		b := l.metrics[minfo.offset]
		if b == nil {
			continue
		}

		allArchived := true
		b.iterFromTo(from, to, func(b *buffer) error {
			if !b.archived {
				allArchived = false
				return fmt.Errorf("stop") // Early termination signal
			}
			return nil
		})

		if allArchived {
			continue
		}

		data := make([]schema.Float, (to-from)/b.frequency+1)
		data, start, end, err := b.read(from, to, data)
		if err != nil {
			return nil, err
		}

		for i := int((end - start) / b.frequency); i < len(data); i++ {
			data[i] = schema.NaN
		}

		retval.Metrics[metric] = &CheckpointMetrics{
			Frequency: b.frequency,
			Start:     start,
			Data:      data,
		}
	}

	for name, child := range l.children {
		val, err := child.toCheckpointFile(from, to, m)
		if err != nil {
			return nil, err
		}

		if val != nil {
			retval.Children[name] = val
		}
	}

	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
		return nil, nil
	}

	return retval, nil
}

// toCheckpoint writes a Level's data to a JSON checkpoint file.
// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.
func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}

	if cf == nil {
		return ErrNoNewArchiveData
	}


   700→	}
   701→	return int(n), successDirs, nil
   702→}
   703→
   704→// toCheckpointBinary writes a binary snapshot file for a single host-level node.
   705→// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
   706→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   707→	cf, err := l.toCheckpointFile(from, to, m)
   708→	if err != nil {
   709→		return err
   710→	}
   711→	if cf == nil {
   712→		return ErrNoNewArchiveData
   713→	}
   714→
   715→	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
   716→		return fmt.Errorf("mkdir %s: %w", dir, err)
   717→	}
   718→
   719→	// Write to a temp file first, then rename (atomic on POSIX).
   720→	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
   721→	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))
   722→
   723→	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   724→	if err != nil {
   725→		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
   726→	}
   727→
   728→	bw := bufio.NewWriter(f)
   729→	if err := writeBinarySnapshotFile(bw, cf); err != nil {
   730→		f.Close()
   731→		os.Remove(tmpPath)
   732→		return fmt.Errorf("write binary snapshot: %w", err)
   733→	}
   734→	if err := bw.Flush(); err != nil {
   735→		f.Close()
   736→		os.Remove(tmpPath)
   737→		return err
   738→	}
   739→	f.Close()
   740→
   741→	return os.Rename(tmpPath, finalPath)
   742→}
   743→
   744→// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
   745→func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
   746→	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
   747→		return err
   748→	}
   749→	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
   750→		return err
   751→	}
   752→	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
   753→		return err
   754→	}
   755→	return writeBinaryLevel(w, cf)
   756→}
   757→
   758→// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
   759→func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
   760→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
   761→		return err
   762→	}
   763→
   764→	for name, metric := range cf.Metrics {
   765→		if err := writeString16(w, name); err != nil {
   766→			return err
   767→		}
   768→		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
   769→			return err
   770→		}
   771→		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
   772→			return err
   773→		}
   774→		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
   775→			return err
   776→		}
   777→		for _, v := range metric.Data {
   778→			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
   779→				return err
   780→			}
   781→		}
   782→	}
   783→
   784→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
   785→		return err
   786→	}
   787→
   788→	for name, child := range cf.Children {
   789→		if err := writeString16(w, name); err != nil {
   790→			return err
   791→		}
   792→		if err := writeBinaryLevel(w, child); err != nil {
   793→			return err
   794→		}
   795→	}
   796→
   797→	return nil
   798→}
   799→

	}
	return int(n), successDirs, nil
}

// toCheckpointBinary writes a binary snapshot file for a single host-level node.
// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}
	if cf == nil {
		return ErrNoNewArchiveData
	}

	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
		return fmt.Errorf("mkdir %s: %w", dir, err)
	}

	// Write to a temp file first, then rename (atomic on POSIX).
	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))

	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
	}

	bw := bufio.NewWriter(f)
	if err := writeBinarySnapshotFile(bw, cf); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return fmt.Errorf("write binary snapshot: %w", err)
	}
	if err := bw.Flush(); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return err
	}
	f.Close()

	return os.Rename(tmpPath, finalPath)
}

// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
		return err
	}
	return writeBinaryLevel(w, cf)
}

// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
		return err
	}

	for name, metric := range cf.Metrics {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
			return err
		}
		for _, v := range metric.Data {
			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
				return err
			}
		}
	}

	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
		return err
	}

	for name, child := range cf.Children {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := writeBinaryLevel(w, child); err != nil {
			return err
		}
	}

	return nil
}


   395→	// Start HTTP server
   396→	wg.Go(func() {
   397→		if err := srv.Start(ctx); err != nil {
   398→			errChan <- err
   399→		}
   400→	})
   401→
   402→	// Handle shutdown signals
   403→	wg.Add(1)
   404→	sigs := make(chan os.Signal, 1)
   405→	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
   406→	go func() {
   407→		defer wg.Done()
   408→		select {
   409→		case <-sigs:
   410→			cclog.Info("Shutdown signal received")
   411→		case <-ctx.Done():
   412→		}
   413→
   414→		runtime.SystemdNotify(false, "Shutting down ...")
   415→		srv.Shutdown(ctx)
   416→		util.FsWatcherShutdown()
   417→		taskmanager.Shutdown()
   418→	}()
   419→
   420→	// Set GC percent if not configured
   421→	if os.Getenv(envGOGC) == "" {
   422→		// trigger GC when heap grows 15% above the previous live set
   423→		debug.SetGCPercent(15)
   424→	}
   425→	runtime.SystemdNotify(true, "running")
   426→
   427→	waitDone := make(chan struct{})
   428→	go func() {
   429→		wg.Wait()
   430→		close(waitDone)
   431→	}()
   432→
   433→	go func() {
   434→		<-waitDone
   435→		close(errChan)
   436→	}()
   437→
   438→	// Wait for either:
   439→	// 1. An error from server startup
   440→	// 2. Completion of all goroutines (normal shutdown or crash)
   441→	select {
   442→	case err := <-errChan:
   443→		// errChan will be closed when waitDone is closed, which happens
   444→		// when all goroutines complete (either from normal shutdown or error)
   445→		if err != nil {
   446→			return err
   447→		}
   448→	case <-time.After(100 * time.Millisecond):
   449→		// Give the server 100ms to start and report any immediate startup errors
   450→		// After that, just wait for normal shutdown completion
   451→		select {
   452→		case err := <-errChan:
   453→			if err != nil {
   454→				return err
   455→			}
   456→		case <-waitDone:
   457→			// Normal shutdown completed
   458→		}
   459→	}
   460→
   461→	cclog.Print("Graceful shutdown completed!")
   462→	return nil
   463→}
   464→

	// Start HTTP server
	wg.Go(func() {
		if err := srv.Start(ctx); err != nil {
			errChan <- err
		}
	})

	// Handle shutdown signals
	wg.Add(1)
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		defer wg.Done()
		select {
		case <-sigs:
			cclog.Info("Shutdown signal received")
		case <-ctx.Done():
		}

		runtime.SystemdNotify(false, "Shutting down ...")
		srv.Shutdown(ctx)
		util.FsWatcherShutdown()
		taskmanager.Shutdown()
	}()

	// Set GC percent if not configured
	if os.Getenv(envGOGC) == "" {
		// trigger GC when heap grows 15% above the previous live set
		debug.SetGCPercent(15)
	}
	runtime.SystemdNotify(true, "running")

	waitDone := make(chan struct{})
	go func() {
		wg.Wait()
		close(waitDone)
	}()

	go func() {
		<-waitDone
		close(errChan)
	}()

	// Wait for either:
	// 1. An error from server startup
	// 2. Completion of all goroutines (normal shutdown or crash)
	select {
	case err := <-errChan:
		// errChan will be closed when waitDone is closed, which happens
		// when all goroutines complete (either from normal shutdown or error)
		if err != nil {
			return err
		}
	case <-time.After(100 * time.Millisecond):
		// Give the server 100ms to start and report any immediate startup errors
		// After that, just wait for normal shutdown completion
		select {
		case err := <-errChan:
			if err != nil {
				return err
			}
		case <-waitDone:
			// Normal shutdown completed
		}
	}

	cclog.Print("Graceful shutdown completed!")
	return nil
}


Now I have a complete picture. Let me create the plan.


Agent

Design shutdown improvement plan

Prompt:
I need a plan to fix a 60-second shutdown timeout issue in a Go HPC monitoring backend. Here's the full context:

## Problem
The shutdown consistently hits the 60s timeout. The log shows:
```
INFO  main.go:410: Shutdown signal received
INFO  client.go:241: NATS connection closed
INFO  archiveWorker.go:223: Initiating archiver shutdown...
INFO  archiveWorker.go:238: All archive jobs completed successfully
INFO  metricstore.go:286: [METRICSTORE]> Writing to './var/metric-checkpoints'...
INFO  archiveWorker.go:121: Archive channel closed, worker exiting
WARN  server.go:448: Shutdown deadline exceeded, forcing exit
```

The archiver finishes fast. The metricstore checkpoint write takes >60s and never completes within the deadline.

## Architecture
The metricstore keeps all metric data in memory (per cluster, per host, per metric). On shutdown it writes binary snapshot files for every host in parallel via a worker pool.

### Key code paths:

**server.go:409-450** - `Server.Shutdown()`:
- Closes NATS, shuts down HTTP (5s timeout)
- Runs metricstore.Shutdown() and archiver.Shutdown() concurrently via WaitGroup
- 60s overall deadline; if exceeded, forces exit

**metricstore.go:273-310** - `Shutdown()`:
1. Cancels context (stops all background goroutines)
2. Closes WAL shard channels
3. Calls `ToCheckpointWAL()` — writes binary snapshots for ALL hosts using NumWorkers worker pool
4. Calls `RotateWALFilesAfterShutdown()` — deletes WAL files

**walCheckpoint.go:627-702** - `ToCheckpointWAL()`:
- Collects all cluster/host pairs
- Spawns NumWorkers goroutines processing a work channel
- Each work item calls `toCheckpointBinary()` which:
  1. Calls `toCheckpointFile()` (holds RLock, reads all metric data for host)
  2. Writes binary file via `writeBinarySnapshotFile()`

**checkpoint.go:261-326** - `toCheckpointFile()`:
- Holds RLock on the Level
- For each metric, checks if all buffers are archived; if so, skips
- Otherwise reads data from `from` to `to` into []Float slice
- Recurses into children (socket → core → etc.)

**WALStaging goroutines** (walCheckpoint.go:154-320):
- N sharded goroutines writing WAL records to per-host files
- On ctx.Done(): drain remaining messages, flush, close files
- On channel close: also exit
- Shutdown() closes these channels but doesn't wait for them to finish

**RotateWALFilesAfterShutdown** (walCheckpoint.go:353-360):
- Simply does os.Remove() on each host's current.wal
- Only called AFTER ToCheckpointWAL succeeds
- If checkpoint times out, WAL files are NEVER deleted

## User Requirements
1. Add log messages to indicate shutdown phases and timings
2. WAL journals MUST be deleted before shutdown completes
3. Understand and improve what takes so long

## Analysis
The bottleneck is `ToCheckpointWAL()` writing binary snapshots for potentially thousands of hosts. Each host requires:
- RLock traversal of entire subtree (socket/core/accelerator levels)
- Data copy from buffer linked list
- Binary serialization and file I/O

Problems:
1. No timing/progress logging during checkpoint — impossible to diagnose
2. WAL deletion happens after checkpoint; if checkpoint times out, WALs remain
3. No way to know how many hosts or how much data is being written
4. The `from` timestamp may be up to 12h old (last periodic checkpoint), meaning massive data ranges

## Design the plan for:

1. **Shutdown phase logging with timings**: Add timing logs for each phase in server.go Shutdown() and metricstore Shutdown(). Include host count and file count progress.

2. **WAL deletion strategy**: Delete each host's WAL file immediately after its binary snapshot succeeds (inside the worker loop), not all at the end. This ensures WAL files are cleaned up even if the overall checkpoint doesn't finish within the deadline.

3. **Progress logging during checkpoint**: In ToCheckpointWAL, log progress periodically (e.g., every 100 hosts or every 10 seconds).

4. **Consider**: Should we wait for WALStaging goroutines to drain before starting checkpoint writes? Currently there's a potential race.

Please provide a detailed implementation plan with specific code changes needed.


I need a plan to fix a 60-second shutdown timeout issue in a Go HPC monitoring backend. Here's the full context:

## Problem
The shutdown consistently hits the 60s timeout. The log shows:
```
INFO  main.go:410: Shutdown signal received
INFO  client.go:241: NATS connection closed
INFO  archiveWorker.go:223: Initiating archiver shutdown...
INFO  archiveWorker.go:238: All archive jobs completed successfully
INFO  metricstore.go:286: [METRICSTORE]> Writing to './var/metric-checkpoints'...
INFO  archiveWorker.go:121: Archive channel closed, worker exiting
WARN  server.go:448: Shutdown deadline exceeded, forcing exit
```

The archiver finishes fast. The metricstore checkpoint write takes >60s and never completes within the deadline.

## Architecture
The metricstore keeps all metric data in memory (per cluster, per host, per metric). On shutdown it writes binary snapshot files for every host in parallel via a worker pool.

### Key code paths:

**server.go:409-450** - `Server.Shutdown()`:
- Closes NATS, shuts down HTTP (5s timeout)
- Runs metricstore.Shutdown() and archiver.Shutdown() concurrently via WaitGroup
- 60s overall deadline; if exceeded, forces exit

**metricstore.go:273-310** - `Shutdown()`:
1. Cancels context (stops all background goroutines)
2. Closes WAL shard channels
3. Calls `ToCheckpointWAL()` — writes binary snapshots for ALL hosts using NumWorkers worker pool
4. Calls `RotateWALFilesAfterShutdown()` — deletes WAL files

**walCheckpoint.go:627-702** - `ToCheckpointWAL()`:
- Collects all cluster/host pairs
- Spawns NumWorkers goroutines processing a work channel
- Each work item calls `toCheckpointBinary()` which:
  1. Calls `toCheckpointFile()` (holds RLock, reads all metric data for host)
  2. Writes binary file via `writeBinarySnapshotFile()`

**checkpoint.go:261-326** - `toCheckpointFile()`:
- Holds RLock on the Level
- For each metric, checks if all buffers are archived; if so, skips
- Otherwise reads data from `from` to `to` into []Float slice
- Recurses into children (socket → core → etc.)

**WALStaging goroutines** (walCheckpoint.go:154-320):
- N sharded goroutines writing WAL records to per-host files
- On ctx.Done(): drain remaining messages, flush, close files
- On channel close: also exit
- Shutdown() closes these channels but doesn't wait for them to finish

**RotateWALFilesAfterShutdown** (walCheckpoint.go:353-360):
- Simply does os.Remove() on each host's current.wal
- Only called AFTER ToCheckpointWAL succeeds
- If checkpoint times out, WAL files are NEVER deleted

## User Requirements
1. Add log messages to indicate shutdown phases and timings
2. WAL journals MUST be deleted before shutdown completes
3. Understand and improve what takes so long

## Analysis
The bottleneck is `ToCheckpointWAL()` writing binary snapshots for potentially thousands of hosts. Each host requires:
- RLock traversal of entire subtree (socket/core/accelerator levels)
- Data copy from buffer linked list
- Binary serialization and file I/O

Problems:
1. No timing/progress logging during checkpoint — impossible to diagnose
2. WAL deletion happens after checkpoint; if checkpoint times out, WALs remain
3. No way to know how many hosts or how much data is being written
4. The `from` timestamp may be up to 12h old (last periodic checkpoint), meaning massive data ranges

## Design the plan for:

1. **Shutdown phase logging with timings**: Add timing logs for each phase in server.go Shutdown() and metricstore Shutdown(). Include host count and file count progress.

2. **WAL deletion strategy**: Delete each host's WAL file immediately after its binary snapshot succeeds (inside the worker loop), not all at the end. This ensures WAL files are cleaned up even if the overall checkpoint doesn't finish within the deadline.

3. **Progress logging during checkpoint**: In ToCheckpointWAL, log progress periodically (e.g., every 100 hosts or every 10 seconds).

4. **Consider**: Should we wait for WALStaging goroutines to drain before starting checkpoint writes? Currently there's a potential race.

Please provide a detailed implementation plan with specific code changes needed.

Prompt:
I need a plan to fix a 60-second shutdown timeout issue in a Go HPC monitoring backend. Here's the full context:

## Problem
The shutdown consistently hits the 60s timeout. The log shows:
```
INFO  main.go:410: Shutdown signal received
INFO  client.go:241: NATS connection closed
INFO  archiveWorker.go:223: Initiating archiver shutdown...
INFO  archiveWorker.go:238: All archive jobs completed successfully
INFO  metricstore.go:286: [METRICSTORE]> Writing to './var/metric-checkpoints'...
INFO  archiveWorker.go:121: Archive channel closed, worker exiting
WARN  server.go:448: Shutdown deadline exceeded, forcing exit
```

The archiver finishes fast. The metricstore checkpoint write takes >60s and never completes within the deadline.

## Architecture
The metricstore keeps all metric data in memory (per cluster, per host, per metric). On shutdown it writes binary snapshot files for every host in parallel via a worker pool.

### Key code paths:

**server.go:409-450** - `Server.Shutdown()`:
- Closes NATS, shuts down HTTP (5s timeout)
- Runs metricstore.Shutdown() and archiver.Shutdown() concurrently via WaitGroup
- 60s overall deadline; if exceeded, forces exit

**metricstore.go:273-310** - `Shutdown()`:
1. Cancels context (stops all background goroutines)
2. Closes WAL shard channels
3. Calls `ToCheckpointWAL()` — writes binary snapshots for ALL hosts using NumWorkers worker pool
4. Calls `RotateWALFilesAfterShutdown()` — deletes WAL files

**walCheckpoint.go:627-702** - `ToCheckpointWAL()`:
- Collects all cluster/host pairs
- Spawns NumWorkers goroutines processing a work channel
- Each work item calls `toCheckpointBinary()` which:
  1. Calls `toCheckpointFile()` (holds RLock, reads all metric data for host)
  2. Writes binary file via `writeBinarySnapshotFile()`

**checkpoint.go:261-326** - `toCheckpointFile()`:
- Holds RLock on the Level
- For each metric, checks if all buffers are archived; if so, skips
- Otherwise reads data from `from` to `to` into []Float slice
- Recurses into children (socket → core → etc.)

**WALStaging goroutines** (walCheckpoint.go:154-320):
- N sharded goroutines writing WAL records to per-host files
- On ctx.Done(): drain remaining messages, flush, close files
- On channel close: also exit
- Shutdown() closes these channels but doesn't wait for them to finish

**RotateWALFilesAfterShutdown** (walCheckpoint.go:353-360):
- Simply does os.Remove() on each host's current.wal
- Only called AFTER ToCheckpointWAL succeeds
- If checkpoint times out, WAL files are NEVER deleted

## User Requirements
1. Add log messages to indicate shutdown phases and timings
2. WAL journals MUST be deleted before shutdown completes
3. Understand and improve what takes so long

## Analysis
The bottleneck is `ToCheckpointWAL()` writing binary snapshots for potentially thousands of hosts. Each host requires:
- RLock traversal of entire subtree (socket/core/accelerator levels)
- Data copy from buffer linked list
- Binary serialization and file I/O

Problems:
1. No timing/progress logging during checkpoint — impossible to diagnose
2. WAL deletion happens after checkpoint; if checkpoint times out, WALs remain
3. No way to know how many hosts or how much data is being written
4. The `from` timestamp may be up to 12h old (last periodic checkpoint), meaning massive data ranges

## Design the plan for:

1. **Shutdown phase logging with timings**: Add timing logs for each phase in server.go Shutdown() and metricstore Shutdown(). Include host count and file count progress.

2. **WAL deletion strategy**: Delete each host's WAL file immediately after its binary snapshot succeeds (inside the worker loop), not all at the end. This ensures WAL files are cleaned up even if the overall checkpoint doesn't finish within the deadline.

3. **Progress logging during checkpoint**: In ToCheckpointWAL, log progress periodically (e.g., every 100 hosts or every 10 seconds).

4. **Consider**: Should we wait for WALStaging goroutines to drain before starting checkpoint writes? Currently there's a potential race.

Please provide a detailed implementation plan with specific code changes needed.


Error: File does not exist. Note: your current working directory is /Users/jan/prg/ClusterCockpit/cc-backend.


   400→		return fmt.Errorf("dropping privileges: %w", err)
   401→	}
   402→
   403→	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
   404→		return fmt.Errorf("server failed: %w", err)
   405→	}
   406→	return nil
   407→}
   408→
   409→func (s *Server) Shutdown(ctx context.Context) {
   410→	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
   411→	defer cancel()
   412→
   413→	nc := nats.GetClient()
   414→	if nc != nil {
   415→		nc.Close()
   416→	}
   417→
   418→	if err := s.server.Shutdown(shutdownCtx); err != nil {
   419→		cclog.Errorf("Server shutdown error: %v", err)
   420→	}
   421→
   422→	// Run metricstore and archiver shutdown concurrently.
   423→	// They are independent: metricstore writes .bin snapshots,
   424→	// archiver flushes pending job archives.
   425→	done := make(chan struct{})
   426→	go func() {
   427→		defer close(done)
   428→		var wg sync.WaitGroup
   429→
   430→		if ms := metricstore.GetMemoryStore(); ms != nil {
   431→			wg.Go(func() {
   432→				metricstore.Shutdown()
   433→			})
   434→		}
   435→
   436→		wg.Go(func() {
   437→			if err := archiver.Shutdown(10 * time.Second); err != nil {
   438→				cclog.Warnf("Archiver shutdown: %v", err)
   439→			}
   440→		})
   441→
   442→		wg.Wait()
   443→	}()
   444→
   445→	select {
   446→	case <-done:
   447→	case <-time.After(60 * time.Second):
   448→		cclog.Warn("Shutdown deadline exceeded, forcing exit")
   449→	}
   450→}
   451→


internal/api/metricstore.go
pkg/metricstore/metricstore_test.go
pkg/metricstore/metricstore.go


   260→// Shutdown performs a graceful shutdown of the metric store.
   261→//
   262→// This function cancels all background goroutines started by Init() and writes
   263→// a final checkpoint to disk before returning. It should be called when the
   264→// application receives a termination signal.
   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→
   290→	ms := GetMemoryStore()
   291→
   292→	lastCheckpointMu.Lock()
   293→	from := lastCheckpoint
   294→	lastCheckpointMu.Unlock()
   295→
   296→	if Keys.Checkpoints.FileFormat == "wal" {
   297→		var hostDirs []string
   298→		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   299→		if err == nil {
   300→			RotateWALFilesAfterShutdown(hostDirs)
   301→		}
   302→	} else {
   303→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   304→	}
   305→
   306→	if err != nil {
   307→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
   308→	}
   309→	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
   310→}
   311→
   312→// Retention starts a background goroutine that periodically frees old metric data.
   313→//
   314→// This worker runs at half the retention interval and calls Free() to remove buffers
   315→// older than the configured retention time. It respects the NodeProvider to preserve
   316→// data for nodes with active jobs.
   317→//
   318→// Parameters:
   319→//   - wg: WaitGroup to signal completion when context is cancelled
   320→//   - ctx: Context for cancellation signal
   321→//
   322→// The goroutine exits when ctx is cancelled.
   323→func Retention(wg *sync.WaitGroup, ctx context.Context) {
   324→	ms := GetMemoryStore()
   325→
   326→	wg.Go(func() {
   327→		d, err := time.ParseDuration(Keys.RetentionInMemory)
   328→		if err != nil {
   329→			cclog.Fatal(err)


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
     7→//
     8→// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
     9→// per host, eliminating the extra memory overhead of the AvroStore and providing
    10→// truly continuous (per-write) crash safety.
    11→//
    12→// # Architecture
    13→//
    14→//	Metric write (DecodeLine)
    15→//	      │
    16→//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
    17→//	      │
    18→//	      └─► WALMessages channel
    19→//	               │
    20→//	               ▼
    21→//	        WALStaging goroutine
    22→//	               │
    23→//	               ▼
    24→//	        checkpoints/cluster/host/current.wal  (append-only, binary)
    25→//
    26→//	Periodic checkpoint (Checkpointing goroutine):
    27→//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
    28→//	  2. Signal WALStaging to truncate current.wal per host
    29→//
    30→//	On restart (FromCheckpoint):
    31→//	  1. Load most recent <timestamp>.bin snapshot
    32→//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
    33→//
    34→// # WAL Record Format
    35→//
    36→//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
    37→//
    38→//	payload:
    39→//	  [8B timestamp int64]
    40→//	  [2B metric_name_len uint16][N metric name bytes]
    41→//	  [1B selector_count uint8]
    42→//	  per selector: [1B selector_len uint8][M selector bytes]
    43→//	  [4B value float32 bits]
    44→//
    45→// # Binary Snapshot Format
    46→//
    47→//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
    48→//	Level tree (recursive):
    49→//	  [4B num_metrics uint32]
    50→//	  per metric:


   100→	Node       string
   101→	Selector   []string
   102→	Value      schema.Float
   103→	Timestamp  int64
   104→}
   105→
   106→// walRotateReq requests WAL file rotation for a specific host directory.
   107→// The done channel is closed by the WAL goroutine when rotation is complete.
   108→type walRotateReq struct {
   109→	hostDir string
   110→	done    chan struct{}
   111→}
   112→
   113→// walFileState holds an open WAL file handle and buffered writer for one host directory.
   114→type walFileState struct {
   115→	f     *os.File
   116→	w     *bufio.Writer
   117→	dirty bool
   118→}
   119→
   120→// walFlushInterval controls how often dirty WAL files are flushed to disk.
   121→// Decoupling flushes from message processing lets the consumer run at memory
   122→// speed, amortizing syscall overhead across many writes.
   123→const walFlushInterval = 5 * time.Second
   124→
   125→// walShardIndex computes which shard a message belongs to based on cluster+node.
   126→// Uses FNV-1a hash for fast, well-distributed mapping.
   127→func walShardIndex(cluster, node string) int {
   128→	h := fnv.New32a()
   129→	h.Write([]byte(cluster))
   130→	h.Write([]byte{0})
   131→	h.Write([]byte(node))
   132→	return int(h.Sum32()) % walNumShards
   133→}
   134→
   135→// SendWALMessage routes a WAL message to the appropriate shard channel.
   136→// Returns false if the channel is full (message dropped).
   137→func SendWALMessage(msg *WALMessage) bool {
   138→	if walShardChs == nil {
   139→		return false
   140→	}
   141→	shard := walShardIndex(msg.Cluster, msg.Node)
   142→	select {
   143→	case walShardChs[shard] <- msg:
   144→		return true
   145→	default:
   146→		return false
   147→	}
   148→}
   149→
   150→// WALStaging starts N sharded background goroutines that receive WALMessage items
   151→// and append binary WAL records to per-host current.wal files.
   152→// Each shard owns a subset of hosts (determined by hash of cluster+node),
   153→// parallelizing serialization and I/O while keeping per-host writes serial.
   154→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   155→	if Keys.Checkpoints.FileFormat == "json" {
   156→		return
   157→	}
   158→
   159→	walNumShards = max(Keys.NumWorkers, 1)
   160→	chBufSize := max(65536/walNumShards, 1024)
   161→
   162→	walShardChs = make([]chan *WALMessage, walNumShards)
   163→	walShardRotateChs = make([]chan walRotateReq, walNumShards)
   164→
   165→	for i := range walNumShards {
   166→		walShardChs[i] = make(chan *WALMessage, chBufSize)
   167→		walShardRotateChs[i] = make(chan walRotateReq, 64)
   168→	}
   169→
   170→	for i := range walNumShards {
   171→		msgCh := walShardChs[i]
   172→		rotateCh := walShardRotateChs[i]
   173→
   174→		wg.Go(func() {
   175→			hostFiles := make(map[string]*walFileState)
   176→
   177→			defer func() {
   178→				for _, ws := range hostFiles {
   179→					if ws.f != nil {
   180→						ws.w.Flush()
   181→						ws.f.Close()
   182→					}
   183→				}
   184→			}()
   185→
   186→			getOrOpenWAL := func(hostDir string) *walFileState {
   187→				ws, ok := hostFiles[hostDir]
   188→				if ok {
   189→					return ws
   190→				}
   191→
   192→				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   193→					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   194→					return nil
   195→				}
   196→
   197→				walPath := path.Join(hostDir, "current.wal")
   198→				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   199→				if err != nil {
   200→					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   201→					return nil
   202→				}
   203→
   204→				w := bufio.NewWriter(f)
   205→
   206→				// Write file header magic if file is new (empty).
   207→				info, err := f.Stat()
   208→				if err == nil && info.Size() == 0 {
   209→					var hdr [4]byte
   210→					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   211→					if _, err := w.Write(hdr[:]); err != nil {
   212→						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   213→						f.Close()
   214→						return nil
   215→					}
   216→				}
   217→
   218→				ws = &walFileState{f: f, w: w}
   219→				hostFiles[hostDir] = ws
   220→				return ws
   221→			}
   222→
   223→			processMsg := func(msg *WALMessage) {
   224→				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   225→				ws := getOrOpenWAL(hostDir)
   226→				if ws == nil {
   227→					return
   228→				}
   229→				if err := writeWALRecordDirect(ws.w, msg); err != nil {
   230→					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   231→				}
   232→				ws.dirty = true
   233→			}
   234→
   235→			processRotate := func(req walRotateReq) {
   236→				ws, ok := hostFiles[req.hostDir]
   237→				if ok && ws.f != nil {
   238→					ws.w.Flush()
   239→					ws.f.Close()
   240→					walPath := path.Join(req.hostDir, "current.wal")
   241→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   242→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   243→					}
   244→					delete(hostFiles, req.hostDir)
   245→				}
   246→				close(req.done)
   247→			}
   248→
   249→			flushDirty := func() {
   250→				for _, ws := range hostFiles {
   251→					if ws.dirty {
   252→						ws.w.Flush()
   253→						ws.dirty = false
   254→					}
   255→				}
   256→			}
   257→
   258→			drain := func() {
   259→				for {
   260→					select {
   261→					case msg, ok := <-msgCh:
   262→						if !ok {
   263→							return
   264→						}
   265→						processMsg(msg)
   266→					case req := <-rotateCh:
   267→						processRotate(req)
   268→					default:
   269→						flushDirty()
   270→						return
   271→					}
   272→				}
   273→			}
   274→
   275→			ticker := time.NewTicker(walFlushInterval)
   276→			defer ticker.Stop()
   277→
   278→			// drainBatch processes up to 4096 pending messages without blocking.
   279→			// Returns false if the channel was closed.
   280→			drainBatch := func() bool {
   281→				for range 4096 {
   282→					select {
   283→					case msg, ok := <-msgCh:
   284→						if !ok {
   285→							flushDirty()
   286→							return false
   287→						}
   288→						processMsg(msg)
   289→					case req := <-rotateCh:
   290→						processRotate(req)
   291→					default:
   292→						return true
   293→					}
   294→				}
   295→				return true
   296→			}
   297→
   298→			for {
   299→				select {
   300→				case <-ctx.Done():
   301→					drain()
   302→					return
   303→				case msg, ok := <-msgCh:
   304→					if !ok {
   305→						return
   306→					}
   307→					processMsg(msg)
   308→					if !drainBatch() {
   309→						return
   310→					}
   311→					// No flush here — timer handles periodic flushing.
   312→				case <-ticker.C:
   313→					flushDirty()
   314→				case req := <-rotateCh:
   315→					processRotate(req)
   316→				}
   317→			}
   318→		})
   319→	}
   320→}
   321→
   322→// RotateWALFiles sends rotation requests for the given host directories
   323→// and blocks until all rotations complete. Each request is routed to the
   324→// shard that owns the host directory.
   325→func RotateWALFiles(hostDirs []string) {
   326→	if walShardRotateChs == nil {
   327→		return
   328→	}
   329→	dones := make([]chan struct{}, len(hostDirs))


   330→	for i, dir := range hostDirs {
   331→		dones[i] = make(chan struct{})
   332→		// Extract cluster/node from hostDir to find the right shard.
   333→		// hostDir = rootDir/cluster/node
   334→		shard := walShardIndexFromDir(dir)
   335→		walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: dones[i]}
   336→	}
   337→	for _, done := range dones {
   338→		<-done
   339→	}
   340→}
   341→
   342→// walShardIndexFromDir extracts cluster and node from a host directory path
   343→// and returns the shard index. The path format is: rootDir/cluster/node
   344→func walShardIndexFromDir(hostDir string) int {
   345→	// hostDir = .../cluster/node — extract last two path components
   346→	node := path.Base(hostDir)
   347→	cluster := path.Base(path.Dir(hostDir))
   348→	return walShardIndex(cluster, node)
   349→}
   350→
   351→// RotateWALFiles sends rotation requests for the given host directories
   352→// and blocks until all rotations complete.
   353→func RotateWALFilesAfterShutdown(hostDirs []string) {
   354→	for _, dir := range hostDirs {
   355→		walPath := path.Join(dir, "current.wal")
   356→		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   357→			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   358→		}
   359→	}
   360→}
   361→
   362→// writeWALRecordDirect encodes a WAL record directly into the bufio.Writer,
   363→// avoiding heap allocations by using a stack-allocated scratch buffer for
   364→// the fixed-size header/trailer and computing CRC inline.
   365→func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
   366→	// Compute payload size.
   367→	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
   368→	for _, s := range msg.Selector {
   369→		payloadSize += 1 + len(s)
   370→	}
   371→
   372→	// Write magic + payload length (8 bytes header).
   373→	var hdr [8]byte
   374→	binary.LittleEndian.PutUint32(hdr[0:4], walRecordMagic)
   375→	binary.LittleEndian.PutUint32(hdr[4:8], uint32(payloadSize))
   376→	if _, err := w.Write(hdr[:]); err != nil {
   377→		return err
   378→	}
   379→
   380→	// We need to compute CRC over the payload as we write it.
   381→	crc := crc32.NewIEEE()
   382→
   383→	// Timestamp (8 bytes).
   384→	var scratch [8]byte
   385→	binary.LittleEndian.PutUint64(scratch[:8], uint64(msg.Timestamp))
   386→	crc.Write(scratch[:8])
   387→	if _, err := w.Write(scratch[:8]); err != nil {
   388→		return err
   389→	}
   390→
   391→	// Metric name length (2 bytes) + metric name.
   392→	binary.LittleEndian.PutUint16(scratch[:2], uint16(len(msg.MetricName)))
   393→	crc.Write(scratch[:2])
   394→	if _, err := w.Write(scratch[:2]); err != nil {
   395→		return err
   396→	}
   397→	nameBytes := []byte(msg.MetricName)
   398→	crc.Write(nameBytes)
   399→	if _, err := w.Write(nameBytes); err != nil {
   400→		return err
   401→	}
   402→
   403→	// Selector count (1 byte).
   404→	scratch[0] = byte(len(msg.Selector))
   405→	crc.Write(scratch[:1])
   406→	if _, err := w.Write(scratch[:1]); err != nil {
   407→		return err
   408→	}
   409→
   410→	// Selectors (1-byte length + bytes each).
   411→	for _, sel := range msg.Selector {
   412→		scratch[0] = byte(len(sel))
   413→		crc.Write(scratch[:1])
   414→		if _, err := w.Write(scratch[:1]); err != nil {
   415→			return err
   416→		}
   417→		selBytes := []byte(sel)
   418→		crc.Write(selBytes)
   419→		if _, err := w.Write(selBytes); err != nil {
   420→			return err
   421→		}
   422→	}
   423→
   424→	// Value (4 bytes, float32 bits).
   425→	binary.LittleEndian.PutUint32(scratch[:4], math.Float32bits(float32(msg.Value)))
   426→	crc.Write(scratch[:4])
   427→	if _, err := w.Write(scratch[:4]); err != nil {
   428→		return err
   429→	}


   600→		if writeErr == nil && b != nb {
   601→			lvl.metrics[minfo.offset] = nb
   602→		}
   603→		// Ignore write errors for timestamps before buffer start (can happen when
   604→		// replaying WAL entries that predate a loaded snapshot's start time).
   605→		lvl.lock.Unlock()
   606→	}
   607→
   608→	return nil
   609→}
   610→
   611→// joinSelector builds a cache key from a selector slice using null bytes as separators.
   612→func joinSelector(sel []string) string {
   613→	if len(sel) == 0 {
   614→		return ""
   615→	}
   616→	var result strings.Builder
   617→	result.WriteString(sel[0])
   618→	for i := 1; i < len(sel); i++ {
   619→		result.WriteString("\x00" + sel[i])
   620→	}
   621→	return result.String()
   622→}
   623→
   624→// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
   625→// Returns the number of files written, the list of host directories that were
   626→// successfully checkpointed (for WAL rotation), and any errors.
   627→func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
   628→	// Collect all cluster/host pairs.
   629→	m.root.lock.RLock()
   630→	totalHosts := 0
   631→	for _, l1 := range m.root.children {
   632→		l1.lock.RLock()
   633→		totalHosts += len(l1.children)
   634→		l1.lock.RUnlock()
   635→	}
   636→	m.root.lock.RUnlock()
   637→
   638→	levels := make([]*Level, 0, totalHosts)
   639→	selectors := make([][]string, 0, totalHosts)
   640→
   641→	m.root.lock.RLock()
   642→	for sel1, l1 := range m.root.children {
   643→		l1.lock.RLock()
   644→		for sel2, l2 := range l1.children {
   645→			levels = append(levels, l2)
   646→			selectors = append(selectors, []string{sel1, sel2})
   647→		}
   648→		l1.lock.RUnlock()
   649→	}
   650→	m.root.lock.RUnlock()
   651→
   652→	type workItem struct {
   653→		level    *Level
   654→		hostDir  string
   655→		selector []string
   656→	}
   657→
   658→	n, errs := int32(0), int32(0)
   659→	var successDirs []string
   660→	var successMu sync.Mutex
   661→
   662→	var wg sync.WaitGroup
   663→	wg.Add(Keys.NumWorkers)
   664→	work := make(chan workItem, Keys.NumWorkers*2)
   665→
   666→	for range Keys.NumWorkers {
   667→		go func() {
   668→			defer wg.Done()
   669→			for wi := range work {
   670→				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
   671→				if err != nil {
   672→					if err == ErrNoNewArchiveData {
   673→						continue
   674→					}
   675→					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
   676→					atomic.AddInt32(&errs, 1)
   677→				} else {
   678→					atomic.AddInt32(&n, 1)
   679→					successMu.Lock()
   680→					successDirs = append(successDirs, wi.hostDir)
   681→					successMu.Unlock()
   682→				}
   683→			}
   684→		}()
   685→	}
   686→
   687→	for i := range levels {
   688→		hostDir := path.Join(dir, path.Join(selectors[i]...))
   689→		work <- workItem{
   690→			level:    levels[i],
   691→			hostDir:  hostDir,
   692→			selector: selectors[i],
   693→		}
   694→	}
   695→	close(work)
   696→	wg.Wait()
   697→
   698→	if errs > 0 {
   699→		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
   700→	}
   701→	return int(n), successDirs, nil
   702→}
   703→
   704→// toCheckpointBinary writes a binary snapshot file for a single host-level node.
   705→// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
   706→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   707→	cf, err := l.toCheckpointFile(from, to, m)
   708→	if err != nil {
   709→		return err
   710→	}
   711→	if cf == nil {
   712→		return ErrNoNewArchiveData
   713→	}
   714→
   715→	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
   716→		return fmt.Errorf("mkdir %s: %w", dir, err)
   717→	}
   718→
   719→	// Write to a temp file first, then rename (atomic on POSIX).


   250→	}
   251→
   252→	close(work)
   253→	wg.Wait()
   254→
   255→	if errs > 0 {
   256→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   257→	}
   258→	return int(n), nil
   259→}
   260→
   261→// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
   262→// Skips metrics that are already archived. Returns nil if no unarchived data exists.
   263→func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
   264→	l.lock.RLock()
   265→	defer l.lock.RUnlock()
   266→
   267→	retval := &CheckpointFile{
   268→		From:     from,
   269→		To:       to,
   270→		Metrics:  make(map[string]*CheckpointMetrics),
   271→		Children: make(map[string]*CheckpointFile),
   272→	}
   273→
   274→	for metric, minfo := range m.Metrics {
   275→		b := l.metrics[minfo.offset]
   276→		if b == nil {
   277→			continue
   278→		}
   279→
   280→		allArchived := true
   281→		b.iterFromTo(from, to, func(b *buffer) error {
   282→			if !b.archived {
   283→				allArchived = false
   284→				return fmt.Errorf("stop") // Early termination signal
   285→			}
   286→			return nil
   287→		})
   288→
   289→		if allArchived {
   290→			continue
   291→		}
   292→
   293→		data := make([]schema.Float, (to-from)/b.frequency+1)
   294→		data, start, end, err := b.read(from, to, data)
   295→		if err != nil {
   296→			return nil, err
   297→		}
   298→
   299→		for i := int((end - start) / b.frequency); i < len(data); i++ {
   300→			data[i] = schema.NaN
   301→		}
   302→
   303→		retval.Metrics[metric] = &CheckpointMetrics{
   304→			Frequency: b.frequency,
   305→			Start:     start,
   306→			Data:      data,
   307→		}
   308→	}
   309→
   310→	for name, child := range l.children {
   311→		val, err := child.toCheckpointFile(from, to, m)
   312→		if err != nil {
   313→			return nil, err
   314→		}
   315→
   316→		if val != nil {
   317→			retval.Children[name] = val
   318→		}
   319→	}
   320→
   321→	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
   322→		return nil, nil
   323→	}
   324→
   325→	return retval, nil
   326→}
   327→
   328→// toCheckpoint writes a Level's data to a JSON checkpoint file.
   329→// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.


pkg/metricstore/metricstore.go:51:	// shutdownFunc stores the context cancellation function created in Init
pkg/metricstore/metricstore.go:53:	shutdownFunc   context.CancelFunc
pkg/metricstore/metricstore.go:54:	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
pkg/metricstore/metricstore.go:182:	shutdownFuncMu.Lock()
pkg/metricstore/metricstore.go:183:	shutdownFunc = shutdown
pkg/metricstore/metricstore.go:184:	shutdownFuncMu.Unlock()
pkg/metricstore/metricstore.go:274:	shutdownFuncMu.Lock()
pkg/metricstore/metricstore.go:275:	defer shutdownFuncMu.Unlock()
pkg/metricstore/metricstore.go:276:	if shutdownFunc != nil {
pkg/metricstore/metricstore.go:277:		shutdownFunc()


pkg/metricstore/metricstore.go:175:	WALStaging(wg, ctx)
pkg/metricstore/walCheckpoint.go:21://	        WALStaging goroutine
pkg/metricstore/walCheckpoint.go:28://	  2. Signal WALStaging to truncate current.wal per host
pkg/metricstore/walCheckpoint.go:86:// Initialized by WALStaging; nil when WAL is not active.
pkg/metricstore/walCheckpoint.go:92:// walNumShards stores the number of shards (set during WALStaging init).
pkg/metricstore/walCheckpoint.go:150:// WALStaging starts N sharded background goroutines that receive WALMessage items
pkg/metricstore/walCheckpoint.go:154:func WALStaging(wg *sync.WaitGroup, ctx context.Context) {


   160→	}
   161→
   162→	// Try to use less memory by forcing a GC run here and then
   163→	// lowering the target percentage. The default of 100 means
   164→	// that only once the ratio of new allocations execeds the
   165→	// previously active heap, a GC is triggered.
   166→	// Forcing a GC here will set the "previously active heap"
   167→	// to a minumum.
   168→	// runtime.GC()
   169→
   170→	ctx, shutdown := context.WithCancel(context.Background())
   171→
   172→	Retention(wg, ctx)
   173→	Checkpointing(wg, ctx)
   174→	CleanUp(wg, ctx)
   175→	WALStaging(wg, ctx)
   176→	MemoryUsageTracker(wg, ctx)
   177→
   178→	// Note: Signal handling has been removed from this function.
   179→	// The caller is responsible for handling shutdown signals and calling
   180→	// the shutdown() function when appropriate.
   181→	// Store the shutdown function for later use by Shutdown()
   182→	shutdownFuncMu.Lock()
   183→	shutdownFunc = shutdown
   184→	shutdownFuncMu.Unlock()
   185→
   186→	if Keys.Subscriptions != nil {
   187→		wg.Go(func() {
   188→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   189→				cclog.Fatal(err)
   190→			}
   191→		})
   192→	}
   193→}
   194→
   195→// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
   196→//
   197→// This function must be called before GetMemoryStore() and can only be called once due to
   198→// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
   199→//


402-	// Handle shutdown signals
403-	wg.Add(1)
404-	sigs := make(chan os.Signal, 1)
405:	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
406-	go func() {
407-		defer wg.Done()
408-		select {
409-		case <-sigs:
410:			cclog.Info("Shutdown signal received")
411-		case <-ctx.Done():
412-		}
413-
414-		runtime.SystemdNotify(false, "Shutting down ...")
415:		srv.Shutdown(ctx)
416:		util.FsWatcherShutdown()
417:		taskmanager.Shutdown()
418-	}()
419-
420-	// Set GC percent if not configured


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-743-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-744-// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
pkg/metricstore/walCheckpoint.go:745:func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
pkg/metricstore/walCheckpoint.go-746-	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-747-		return err


   704→// toCheckpointBinary writes a binary snapshot file for a single host-level node.
   705→// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
   706→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   707→	cf, err := l.toCheckpointFile(from, to, m)
   708→	if err != nil {
   709→		return err
   710→	}
   711→	if cf == nil {
   712→		return ErrNoNewArchiveData
   713→	}
   714→
   715→	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
   716→		return fmt.Errorf("mkdir %s: %w", dir, err)
   717→	}
   718→
   719→	// Write to a temp file first, then rename (atomic on POSIX).
   720→	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
   721→	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))
   722→
   723→	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   724→	if err != nil {
   725→		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
   726→	}
   727→
   728→	bw := bufio.NewWriter(f)
   729→	if err := writeBinarySnapshotFile(bw, cf); err != nil {
   730→		f.Close()
   731→		os.Remove(tmpPath)
   732→		return fmt.Errorf("write binary snapshot: %w", err)
   733→	}
   734→	if err := bw.Flush(); err != nil {
   735→		f.Close()
   736→		os.Remove(tmpPath)
   737→		return err
   738→	}
   739→	f.Close()
   740→
   741→	return os.Rename(tmpPath, finalPath)
   742→}
   743→
   744→// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
   745→func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
   746→	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
   747→		return err
   748→	}
   749→	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
   750→		return err
   751→	}
   752→	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
   753→		return err


   130→		dec.DisallowUnknownFields()
   131→		if err := dec.Decode(&Keys); err != nil {
   132→			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
   133→		}
   134→	}
   135→
   136→	// Set NumWorkers from config or use default
   137→	if Keys.NumWorkers <= 0 {
   138→		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
   139→	}
   140→	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
   141→
   142→	// Pass the config.MetricStoreKeys
   143→	InitMetrics(metrics)
   144→
   145→	ms := GetMemoryStore()
   146→
   147→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   148→	if err != nil {
   149→		cclog.Fatal(err)
   150→	}
   151→
   152→	restoreFrom := startupTime.Add(-d)
   153→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   154→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   155→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   156→	if err != nil {
   157→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   158→	} else {
   159→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   160→	}
   161→
   162→	// Try to use less memory by forcing a GC run here and then
   163→	// lowering the target percentage. The default of 100 means
   164→	// that only once the ratio of new allocations execeds the


   415→		srv.Shutdown(ctx)
   416→		util.FsWatcherShutdown()
   417→		taskmanager.Shutdown()
   418→	}()
   419→
   420→	// Set GC percent if not configured
   421→	if os.Getenv(envGOGC) == "" {
   422→		// trigger GC when heap grows 15% above the previous live set
   423→		debug.SetGCPercent(15)
   424→	}
   425→	runtime.SystemdNotify(true, "running")
   426→
   427→	waitDone := make(chan struct{})
   428→	go func() {
   429→		wg.Wait()
   430→		close(waitDone)
   431→	}()
   432→
   433→	go func() {
   434→		<-waitDone
   435→		close(errChan)
   436→	}()
   437→
   438→	// Wait for either:
   439→	// 1. An error from server startup


    80→	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
    81→	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
    82→	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
    83→)
    84→
    85→// walShardChs holds per-shard channels for WAL messages.
    86→// Initialized by WALStaging; nil when WAL is not active.
    87→var walShardChs []chan *WALMessage
    88→
    89→// walShardRotateChs holds per-shard channels for WAL rotation requests.
    90→var walShardRotateChs []chan walRotateReq
    91→
    92→// walNumShards stores the number of shards (set during WALStaging init).
    93→var walNumShards int
    94→
    95→// WALMessage represents a single metric write to be appended to the WAL.
    96→// Cluster and Node are NOT stored in the WAL record (inferred from file path).
    97→type WALMessage struct {
    98→	MetricName string
    99→	Cluster    string