The changes yesterday caused a severe Regression on the startup time. The checkpoint loading takes much longer. Before: [INFO]     metricstore.go:159: [METRICSTORE]> Checkpoints loaded (5811 files, 22242 MB, that took 13.982486s)
3/24/2026, 7:19:34 AMINFO[INFO]     metricstore.go:153: [METRICSTORE]> Loading checkpoints newer than 2026-03-23T07:19:34+01:00. And after the changes: etricstore.go:159: [METRICSTORE]> Checkpoints loaded (5860 files, 21338 MB, that took 302.524814s)
3/24/2026, 6:10:53 PMINFO[INFO]     metricstore.go:153: [METRICSTORE]> Loading checkpoints newer than 2026-03-23T18:10:53+01:00
3/24/2026, 6:10:43 PMINFO[INFO]     metricstore.go:153: [METRICSTORE]> Loading checkpoints newer than 2026-03-23T18:10:43+01:00
3/24/2026, 6:09:50 PMINFO[INFO]     metricstore.go:153: [METRICSTORE]> Loading checkpoints newer than 2026-03-23T18:09:50+01:00
3/24/2026, 6:05:57 PMINFO[INFO]     metricstore.go:153: [METRICSTORE]> Loading checkpoints newer than 2026-03-23T18:05:57+01:00
3/24/2026, 6:02:12 PMINFO[INFO]     metricstore.go:153: [METRICSTORE]> Loading checkpoints newer than 2026-03-23T18:02:12+01:00. Investigate the issue and fix it.


Let me investigate the recent changes to the checkpoint loading code.


Bash

git log --oneline -20 --all -- pkg/metricstore/

Recent commits touching metricstore


git diff HEAD~5..HEAD -- pkg/metricstore/ --stat

Files changed in recent commits


6f7dda53 Cleanup
0325d9e8 fix: Increase throughput for WAL writers
e41d1251 fix: Continue on error
586c9020 Restructure metricstore cleanup archiving to stay withinh 32k parquet-go limit
96a79098 carry forward: uncommitted session files
c0d2d65f Introduce average resampler support
09501df3 fix: reduce memory usage in parquet checkpoint archiver
d3b94c33 Completed 'Explore' agent: Find CUDA/HIP macro switching (toolu_01EbN3)
8b132ed7 fix: Blocking ReceiveNats call
bf1a8a17 fix: Shard WAL consumer for higher throughput
50aed595 fix: metricstore NATS contention
818a05f2 carry forward: uncommitted session files
d46e6371 Add log about checkpoint archiving
02f82c2c fix: Prevent memory spikes in parquet writer for  metricstore move policy
0a4c4d8e Patching parquet archive writer for high memory usage
bab6eb4c Convert Warn message on missing metrics to Debug level
39ab1278 Make checkpointInterval an option config option again.
b214e175 Add buffered I/O to WAL writes and fix MemoryCap comment
8234ad31 fix: Fix metricstore memory explosion from broken emergency free and batch aborts
ecd29b28 carry forward: uncommitted session files


diff --git a/pkg/metricstore/walCheckpoint.go b/pkg/metricstore/walCheckpoint.go
index 38585cf5..57064e5f 100644
--- a/pkg/metricstore/walCheckpoint.go
+++ b/pkg/metricstore/walCheckpoint.go
@@ -69,6 +69,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"time"

 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
 	"github.com/ClusterCockpit/cc-lib/v2/schema"
@@ -111,10 +112,16 @@ type walRotateReq struct {

 // walFileState holds an open WAL file handle and buffered writer for one host directory.
 type walFileState struct {
-	f *os.File
-	w *bufio.Writer
+	f     *os.File
+	w     *bufio.Writer
+	dirty bool
 }

+// walFlushInterval controls how often dirty WAL files are flushed to disk.
+// Decoupling flushes from message processing lets the consumer run at memory
+// speed, amortizing syscall overhead across many writes.
+const walFlushInterval = 5 * time.Second
+
 // walShardIndex computes which shard a message belongs to based on cluster+node.
 // Uses FNV-1a hash for fast, well-distributed mapping.
 func walShardIndex(cluster, node string) int {
@@ -222,6 +229,7 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 				if err := writeWALRecordDirect(ws.w, msg); err != nil {
 					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
 				}
+				ws.dirty = true
 			}

 			processRotate := func(req walRotateReq) {
@@ -238,10 +246,11 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 				close(req.done)
 			}

-			flushAll := func() {
+			flushDirty := func() {
 				for _, ws := range hostFiles {
-					if ws.f != nil {
+					if ws.dirty {
 						ws.w.Flush()
+						ws.dirty = false
 					}
 				}
 			}
@@ -257,12 +266,35 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 					case req := <-rotateCh:
 						processRotate(req)
 					default:
-						flushAll()
+						flushDirty()
 						return
 					}
 				}
 			}

+			ticker := time.NewTicker(walFlushInterval)
+			defer ticker.Stop()
+
+			// drainBatch processes up to 4096 pending messages without blocking.
+			// Returns false if the channel was closed.
+			drainBatch := func() bool {
+				for range 4096 {
+					select {
+					case msg, ok := <-msgCh:
+						if !ok {
+							flushDirty()
+							return false
+						}
+						processMsg(msg)
+					case req := <-rotateCh:
+						processRotate(req)
+					default:
+						return true
+					}
+				}
+				return true
+			}
+
 			for {
 				select {
 				case <-ctx.Done():
@@ -273,23 +305,12 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 						return
 					}
 					processMsg(msg)
-
-					// Drain up to 256 more messages without blocking to batch writes.
-					for range 256 {
-						select {
-						case msg, ok := <-msgCh:
-							if !ok {
-								return
-							}
-							processMsg(msg)
-						case req := <-rotateCh:
-							processRotate(req)
-						default:
-							goto flushed
-						}
+					if !drainBatch() {
+						return
 					}
-				flushed:
-					flushAll()
+					// No flush here — timer handles periodic flushing.
+				case <-ticker.C:
+					flushDirty()
 				case req := <-rotateCh:
 					processRotate(req)
 				}
@@ -413,69 +434,6 @@ func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
 	return err
 }

-// buildWALPayload encodes a WALMessage into a binary payload (without magic/length/CRC).
-func buildWALPayload(msg *WALMessage) []byte {
-	size := 8 + 2 + len(msg.MetricName) + 1 + 4
-	for _, s := range msg.Selector {
-		size += 1 + len(s)
-	}
-
-	buf := make([]byte, 0, size)
-
-	// Timestamp (8 bytes, little-endian int64)
-	var ts [8]byte
-	binary.LittleEndian.PutUint64(ts[:], uint64(msg.Timestamp))
-	buf = append(buf, ts[:]...)
-
-	// Metric name (2-byte length prefix + bytes)
-	var mLen [2]byte
-	binary.LittleEndian.PutUint16(mLen[:], uint16(len(msg.MetricName)))
-	buf = append(buf, mLen[:]...)
-	buf = append(buf, msg.MetricName...)
-
-	// Selector count (1 byte)
-	buf = append(buf, byte(len(msg.Selector)))
-
-	// Selectors (1-byte length prefix + bytes each)
-	for _, sel := range msg.Selector {
-		buf = append(buf, byte(len(sel)))
-		buf = append(buf, sel...)
-	}
-
-	// Value (4 bytes, float32 bit representation)
-	var val [4]byte
-	binary.LittleEndian.PutUint32(val[:], math.Float32bits(float32(msg.Value)))
-	buf = append(buf, val[:]...)
-
-	return buf
-}
-
-// writeWALRecord appends a binary WAL record to the writer.
-// Format: [4B magic][4B payload_len][payload][4B CRC32]
-func writeWALRecord(w io.Writer, msg *WALMessage) error {
-	payload := buildWALPayload(msg)
-	crc := crc32.ChecksumIEEE(payload)
-
-	record := make([]byte, 0, 4+4+len(payload)+4)
-
-	var magic [4]byte
-	binary.LittleEndian.PutUint32(magic[:], walRecordMagic)
-	record = append(record, magic[:]...)
-
-	var pLen [4]byte
-	binary.LittleEndian.PutUint32(pLen[:], uint32(len(payload)))
-	record = append(record, pLen[:]...)
-
-	record = append(record, payload...)
-
-	var crcBytes [4]byte
-	binary.LittleEndian.PutUint32(crcBytes[:], crc)
-	record = append(record, crcBytes[:]...)
-
-	_, err := w.Write(record)
-	return err
-}
-
 // readWALRecord reads one WAL record from the reader.
 // Returns (nil, nil) on clean EOF. Returns error on data corruption.
 // A CRC mismatch indicates a truncated trailing record (expected on crash).


git log --oneline -5 -- pkg/metricstore/walCheckpoint.go

Recent commits to walCheckpoint.go


6f7dda53 Cleanup
0325d9e8 fix: Increase throughput for WAL writers
bf1a8a17 fix: Shard WAL consumer for higher throughput
50aed595 fix: metricstore NATS contention
b214e175 Add buffered I/O to WAL writes and fix MemoryCap comment


git diff HEAD~3..HEAD -- pkg/metricstore/checkpoint.go

Changes to checkpoint.go loading logic


(Bash completed with no output)


The WAL writing changes look fine — they only affect write path, not loading. Let me check the loading code more carefully.


git log --all --oneline -10 -- pkg/metricstore/checkpoint.go

Recent commits to checkpoint.go


96a79098 carry forward: uncommitted session files
d3b94c33 Completed 'Explore' agent: Find CUDA/HIP macro switching (toolu_01EbN3)
818a05f2 carry forward: uncommitted session files
39ab1278 Make checkpointInterval an option config option again.
ecd29b28 carry forward: uncommitted session files
21b6c1b5 carry forward: uncommitted session files
39635ea1 Cleanup metricstore options
a418abc7 Run go fix
ca0f9a42 Introduce metric store binary checkpoints with write ahead log
9fc1836c Replace avro checkpoints with native binary format


git diff HEAD~10..HEAD -- pkg/metricstore/checkpoint.go pkg/metricstore/metricstore.go

Recent changes to checkpoint loading


No changes to checkpoint.go or metricstore.go in recent commits. Let me look at what changed between the working and broken state more carefully.


git log --oneline --all --since="2026-03-23" -- pkg/metricstore/

All recent metricstore changes


6f7dda53 Cleanup
0325d9e8 fix: Increase throughput for WAL writers
e41d1251 fix: Continue on error
586c9020 Restructure metricstore cleanup archiving to stay withinh 32k parquet-go limit


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements checkpoint persistence for the in-memory metric store.
     7→//
     8→// Checkpoints enable graceful restarts by periodically saving in-memory metric
     9→// data to disk in JSON or binary format. The checkpoint system:
    10→//
    11→// Key Features:
    12→//   - Periodic background checkpointing via the Checkpointing() worker
    13→//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
    14→//   - Parallel checkpoint creation and loading using worker pools
    15→//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
    16→//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
    17→//   - Only saves unarchived data (archived data is already persisted elsewhere)
    18→//   - GC optimization during loading to prevent excessive heap growth
    19→//
    20→// Checkpoint Workflow:
    21→//  1. Init() loads checkpoints within retention window at startup
    22→//  2. Checkpointing() worker periodically saves new data
    23→//  3. Shutdown() writes final checkpoint before exit
    24→//
    25→// File Organization:
    26→//
    27→//	checkpoints/
    28→//	  cluster1/
    29→//	    host001/
    30→//	      1234567890.json  (JSON format: full subtree snapshot)
    31→//	      1234567890.bin   (binary format: full subtree snapshot)
    32→//	      current.wal      (WAL format: append-only per-entry log)
    33→//	    host002/
    34→//	      ...
    35→package metricstore
    36→
    37→import (
    38→	"bufio"
    39→	"context"
    40→	"encoding/json"
    41→	"errors"
    42→	"fmt"
    43→	"io/fs"
    44→	"os"
    45→	"path"
    46→	"path/filepath"
    47→	"sort"
    48→	"strconv"
    49→	"strings"
    50→	"sync"

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements checkpoint persistence for the in-memory metric store.
//
// Checkpoints enable graceful restarts by periodically saving in-memory metric
// data to disk in JSON or binary format. The checkpoint system:
//
// Key Features:
//   - Periodic background checkpointing via the Checkpointing() worker
//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
//   - Parallel checkpoint creation and loading using worker pools
//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
//   - Only saves unarchived data (archived data is already persisted elsewhere)
//   - GC optimization during loading to prevent excessive heap growth
//
// Checkpoint Workflow:
//  1. Init() loads checkpoints within retention window at startup
//  2. Checkpointing() worker periodically saves new data
//  3. Shutdown() writes final checkpoint before exit
//
// File Organization:
//
//	checkpoints/
//	  cluster1/
//	    host001/
//	      1234567890.json  (JSON format: full subtree snapshot)
//	      1234567890.bin   (binary format: full subtree snapshot)
//	      current.wal      (WAL format: append-only per-entry log)
//	    host002/
//	      ...
package metricstore

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"


    50→	"sync"
    51→	"sync/atomic"
    52→	"time"
    53→
    54→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    55→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    56→)
    57→
    58→const (
    59→	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
    60→	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
    61→	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
    62→)
    63→
    64→// CheckpointMetrics represents metric data in a checkpoint file.
    65→// Whenever the structure changes, update MarshalJSON as well!
    66→type CheckpointMetrics struct {
    67→	Data      []schema.Float `json:"data"`
    68→	Frequency int64          `json:"frequency"`
    69→	Start     int64          `json:"start"`
    70→}
    71→
    72→// CheckpointFile represents the hierarchical structure of a checkpoint file.
    73→// It mirrors the Level tree structure from the MemoryStore.
    74→type CheckpointFile struct {
    75→	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
    76→	Children map[string]*CheckpointFile    `json:"children"`
    77→	From     int64                         `json:"from"`
    78→	To       int64                         `json:"to"`
    79→}
    80→
    81→// lastCheckpoint tracks the timestamp of the last checkpoint creation.
    82→var (
    83→	lastCheckpoint   time.Time
    84→	lastCheckpointMu sync.Mutex
    85→)
    86→
    87→// Checkpointing starts a background worker that periodically saves metric data to disk.
    88→//
    89→// Checkpoints are written every 12 hours (hardcoded).
    90→//
    91→// Format behaviour:
    92→//   - "json": Periodic checkpointing every checkpointInterval
    93→//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
    94→func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
    95→	lastCheckpointMu.Lock()
    96→	lastCheckpoint = time.Now()
    97→	lastCheckpointMu.Unlock()
    98→
    99→	ms := GetMemoryStore()
   100→
   101→	wg.Go(func() {
   102→
   103→		d := 12 * time.Hour // default checkpoint interval
   104→		if Keys.CheckpointInterval != "" {
   105→			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
   106→			if err != nil {
   107→				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
   108→			} else {
   109→				d = parsed
   110→			}
   111→		}
   112→
   113→		ticker := time.NewTicker(d)
   114→		defer ticker.Stop()
   115→
   116→		for {
   117→			select {
   118→			case <-ctx.Done():
   119→				return
   120→			case <-ticker.C:
   121→				lastCheckpointMu.Lock()
   122→				from := lastCheckpoint
   123→				lastCheckpointMu.Unlock()
   124→
   125→				now := time.Now()
   126→				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
   127→
   128→				if Keys.Checkpoints.FileFormat == "wal" {
   129→					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   130→					if err != nil {
   131→						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
   132→					} else {
   133→						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
   134→						lastCheckpointMu.Lock()
   135→						lastCheckpoint = now
   136→						lastCheckpointMu.Unlock()
   137→						// Rotate WAL files for successfully checkpointed hosts.
   138→						RotateWALFiles(hostDirs)
   139→					}
   140→				} else {
   141→					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   142→					if err != nil {
   143→						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
   144→					} else {
   145→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
   146→						lastCheckpointMu.Lock()
   147→						lastCheckpoint = now
   148→						lastCheckpointMu.Unlock()
   149→					}

	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

const (
	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
)

// CheckpointMetrics represents metric data in a checkpoint file.
// Whenever the structure changes, update MarshalJSON as well!
type CheckpointMetrics struct {
	Data      []schema.Float `json:"data"`
	Frequency int64          `json:"frequency"`
	Start     int64          `json:"start"`
}

// CheckpointFile represents the hierarchical structure of a checkpoint file.
// It mirrors the Level tree structure from the MemoryStore.
type CheckpointFile struct {
	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
	Children map[string]*CheckpointFile    `json:"children"`
	From     int64                         `json:"from"`
	To       int64                         `json:"to"`
}

// lastCheckpoint tracks the timestamp of the last checkpoint creation.
var (
	lastCheckpoint   time.Time
	lastCheckpointMu sync.Mutex
)

// Checkpointing starts a background worker that periodically saves metric data to disk.
//
// Checkpoints are written every 12 hours (hardcoded).
//
// Format behaviour:
//   - "json": Periodic checkpointing every checkpointInterval
//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
	lastCheckpointMu.Lock()
	lastCheckpoint = time.Now()
	lastCheckpointMu.Unlock()

	ms := GetMemoryStore()

	wg.Go(func() {

		d := 12 * time.Hour // default checkpoint interval
		if Keys.CheckpointInterval != "" {
			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
			if err != nil {
				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
			} else {
				d = parsed
			}
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				lastCheckpointMu.Lock()
				from := lastCheckpoint
				lastCheckpointMu.Unlock()

				now := time.Now()
				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))

				if Keys.Checkpoints.FileFormat == "wal" {
					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
					} else {
						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
						// Rotate WAL files for successfully checkpointed hosts.
						RotateWALFiles(hostDirs)
					}
				} else {
					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
					}


pkg/metricstore/checkpoint.go:464:func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
pkg/metricstore/parquetArchive_test.go:261:func TestLoadCheckpointFileFromDisk_JSON(t *testing.T) {
pkg/metricstore/parquetArchive.go:194:func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
pkg/metricstore/parquetArchive.go:243:func loadCheckpointFiles(dir string, from int64) ([]*CheckpointFile, []string, error) {


pkg/metricstore/metricstore.go:159:		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())


   140→	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
   141→
   142→	// Pass the config.MetricStoreKeys
   143→	InitMetrics(metrics)
   144→
   145→	ms := GetMemoryStore()
   146→
   147→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   148→	if err != nil {
   149→		cclog.Fatal(err)
   150→	}
   151→
   152→	restoreFrom := startupTime.Add(-d)
   153→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   154→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   155→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   156→	if err != nil {
   157→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   158→	} else {
   159→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   160→	}
   161→
   162→	// Try to use less memory by forcing a GC run here and then
   163→	// lowering the target percentage. The default of 100 means
   164→	// that only once the ratio of new allocations execeds the
   165→	// previously active heap, a GC is triggered.
   166→	// Forcing a GC here will set the "previously active heap"
   167→	// to a minumum.
   168→	// runtime.GC()
   169→
   170→	ctx, shutdown := context.WithCancel(context.Background())
   171→
   172→	Retention(wg, ctx)
   173→	Checkpointing(wg, ctx)
   174→	CleanUp(wg, ctx)
   175→	WALStaging(wg, ctx)
   176→	MemoryUsageTracker(wg, ctx)
   177→
   178→	// Note: Signal handling has been removed from this function.
   179→	// The caller is responsible for handling shutdown signals and calling

	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)

	// Pass the config.MetricStoreKeys
	InitMetrics(metrics)

	ms := GetMemoryStore()

	d, err := time.ParseDuration(Keys.RetentionInMemory)
	if err != nil {
		cclog.Fatal(err)
	}

	restoreFrom := startupTime.Add(-d)
	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
	if err != nil {
		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
	} else {
		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
	}

	// Try to use less memory by forcing a GC run here and then
	// lowering the target percentage. The default of 100 means
	// that only once the ratio of new allocations execeds the
	// previously active heap, a GC is triggered.
	// Forcing a GC here will set the "previously active heap"
	// to a minumum.
	// runtime.GC()

	ctx, shutdown := context.WithCancel(context.Background())

	Retention(wg, ctx)
	Checkpointing(wg, ctx)
	CleanUp(wg, ctx)
	WALStaging(wg, ctx)
	MemoryUsageTracker(wg, ctx)

	// Note: Signal handling has been removed from this function.
	// The caller is responsible for handling shutdown signals and calling


pkg/metricstore/checkpoint.go:395:func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
pkg/metricstore/checkpoint.go:434:func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {


   395→func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
   396→	var wg sync.WaitGroup
   397→	work := make(chan [2]string, Keys.NumWorkers*4)
   398→	n, errs := int32(0), int32(0)
   399→
   400→	wg.Add(Keys.NumWorkers)
   401→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   402→		go func() {
   403→			defer wg.Done()
   404→			for host := range work {
   405→				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
   406→				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
   407→				if err != nil {
   408→					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
   409→					atomic.AddInt32(&errs, 1)
   410→				}
   411→				atomic.AddInt32(&n, int32(nn))
   412→			}
   413→		}()
   414→	}
   415→
   416→	err := enqueueCheckpointHosts(dir, work)
   417→	close(work)
   418→	wg.Wait()
   419→
   420→	if err != nil {
   421→		return int(n), err
   422→	}
   423→
   424→	if errs > 0 {
   425→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   426→	}
   427→	return int(n), nil
   428→}
   429→
   430→// FromCheckpointFiles is the main entry point for loading checkpoints at startup.
   431→//
   432→// Creates checkpoint directory if it doesn't exist. This function must be called
   433→// before any writes or reads, and can only be called once.
   434→func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
   435→	if _, err := os.Stat(dir); os.IsNotExist(err) {
   436→		err := os.MkdirAll(dir, CheckpointDirPerms)
   437→		if err != nil {
   438→			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
   439→		}
   440→		cclog.Debugf("[METRICSTORE]> %#v Directory created successfully", dir)
   441→	}
   442→
   443→	return m.FromCheckpoint(dir, from)
   444→}
   445→
   446→func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
   447→	br := bufio.NewReader(f)
   448→	cf := &CheckpointFile{}
   449→	if err := json.NewDecoder(br).Decode(cf); err != nil {
   450→		return err
   451→	}
   452→
   453→	if cf.To != 0 && cf.To < from {
   454→		return nil
   455→	}
   456→
   457→	if err := l.loadFile(cf, m); err != nil {
   458→		return err
   459→	}
   460→
   461→	return nil
   462→}
   463→
   464→func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
   465→	for name, metric := range cf.Metrics {
   466→		n := len(metric.Data)
   467→		b := &buffer{
   468→			frequency: metric.Frequency,
   469→			start:     metric.Start,
   470→			data:      metric.Data[0:n:n],
   471→			prev:      nil,
   472→			next:      nil,
   473→			archived:  true,
   474→		}
   475→
   476→		minfo, ok := m.Metrics[name]
   477→		if !ok {
   478→			continue
   479→		}
   480→
   481→		prev := l.metrics[minfo.offset]
   482→		if prev == nil {
   483→			l.metrics[minfo.offset] = b
   484→		} else {
   485→			if prev.start > b.start {
   486→				return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start)
   487→			}
   488→
   489→			b.prev = prev
   490→			prev.next = b
   491→		}
   492→		l.metrics[minfo.offset] = b
   493→	}
   494→
   495→	if len(cf.Children) > 0 && l.children == nil {
   496→		l.children = make(map[string]*Level)
   497→	}
   498→
   499→	for sel, childCf := range cf.Children {
   500→		child, ok := l.children[sel]
   501→		if !ok {
   502→			child = &Level{
   503→				metrics:  make([]*buffer, len(m.Metrics)),
   504→				children: nil,
   505→			}
   506→			l.children[sel] = child
   507→		}
   508→
   509→		if err := child.loadFile(childCf, m); err != nil {
   510→			return err
   511→		}
   512→	}
   513→
   514→	return nil

func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
	var wg sync.WaitGroup
	work := make(chan [2]string, Keys.NumWorkers*4)
	n, errs := int32(0), int32(0)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for host := range work {
				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
					atomic.AddInt32(&errs, 1)
				}
				atomic.AddInt32(&n, int32(nn))
			}
		}()
	}

	err := enqueueCheckpointHosts(dir, work)
	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}

	if errs > 0 {
		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
	}
	return int(n), nil
}

// FromCheckpointFiles is the main entry point for loading checkpoints at startup.
//
// Creates checkpoint directory if it doesn't exist. This function must be called
// before any writes or reads, and can only be called once.
func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, CheckpointDirPerms)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
		}
		cclog.Debugf("[METRICSTORE]> %#v Directory created successfully", dir)
	}

	return m.FromCheckpoint(dir, from)
}

func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)
	cf := &CheckpointFile{}
	if err := json.NewDecoder(br).Decode(cf); err != nil {
		return err
	}

	if cf.To != 0 && cf.To < from {
		return nil
	}

	if err := l.loadFile(cf, m); err != nil {
		return err
	}

	return nil
}

func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
	for name, metric := range cf.Metrics {
		n := len(metric.Data)
		b := &buffer{
			frequency: metric.Frequency,
			start:     metric.Start,
			data:      metric.Data[0:n:n],
			prev:      nil,
			next:      nil,
			archived:  true,
		}

		minfo, ok := m.Metrics[name]
		if !ok {
			continue
		}

		prev := l.metrics[minfo.offset]
		if prev == nil {
			l.metrics[minfo.offset] = b
		} else {
			if prev.start > b.start {
				return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start)
			}

			b.prev = prev
			prev.next = b
		}
		l.metrics[minfo.offset] = b
	}

	if len(cf.Children) > 0 && l.children == nil {
		l.children = make(map[string]*Level)
	}

	for sel, childCf := range cf.Children {
		child, ok := l.children[sel]
		if !ok {
			child = &Level{
				metrics:  make([]*buffer, len(m.Metrics)),
				children: nil,
			}
			l.children[sel] = child
		}

		if err := child.loadFile(childCf, m); err != nil {
			return err
		}
	}

	return nil


Now let me look at the `fromCheckpoint` function (lowercase) which is what actually loads files per host:


pkg/metricstore/checkpoint.go:520:func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {


   520→func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {
   521→	direntries, err := os.ReadDir(dir)
   522→	if err != nil {
   523→		if os.IsNotExist(err) {
   524→			return 0, nil
   525→		}
   526→		return 0, err
   527→	}
   528→
   529→	allFiles := make([]fs.DirEntry, 0)
   530→	var walEntry fs.DirEntry
   531→	filesLoaded := 0
   532→
   533→	for _, e := range direntries {
   534→		if e.IsDir() {
   535→			// Legacy: skip subdirectories (only used by old Avro format).
   536→			// These are ignored; their data is not loaded.
   537→			cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir)
   538→			continue
   539→		}
   540→
   541→		name := e.Name()
   542→		if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") {
   543→			allFiles = append(allFiles, e)
   544→		} else if name == "current.wal" {
   545→			walEntry = e
   546→		}
   547→		// Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes).
   548→	}
   549→
   550→	files, err := findFiles(allFiles, from, true)
   551→	if err != nil {
   552→		return filesLoaded, err
   553→	}
   554→
   555→	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
   556→		".json": l.loadJSONFile,
   557→		".bin":  l.loadBinaryFile,
   558→	}
   559→
   560→	for _, filename := range files {
   561→		ext := filepath.Ext(filename)
   562→		loader := loaders[ext]
   563→		if loader == nil {
   564→			cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename)
   565→			continue
   566→		}
   567→
   568→		err := func() error {
   569→			f, err := os.Open(path.Join(dir, filename))
   570→			if err != nil {
   571→				return err
   572→			}
   573→			defer f.Close()
   574→			return loader(m, f, from)
   575→		}()
   576→		if err != nil {
   577→			return filesLoaded, err
   578→		}
   579→		filesLoaded++
   580→	}
   581→
   582→	// Replay WAL after all snapshot files so it fills in data since the last snapshot.
   583→	if walEntry != nil {
   584→		err := func() error {
   585→			f, err := os.Open(path.Join(dir, walEntry.Name()))
   586→			if err != nil {
   587→				return err
   588→			}
   589→			defer f.Close()
   590→			return l.loadWALFile(m, f, from)
   591→		}()
   592→		if err != nil {
   593→			// WAL errors are non-fatal: the snapshot already loaded the bulk of data.
   594→			cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err)
   595→		} else {
   596→			filesLoaded++
   597→		}
   598→	}
   599→
   600→	return filesLoaded, nil
   601→}
   602→
   603→// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename.
   604→// Supports ".json" (format: "<ts>.json") and ".bin" (format: "<ts>.bin").
   605→func parseTimestampFromFilename(name string) (int64, error) {
   606→	switch {
   607→	case strings.HasSuffix(name, ".json"):
   608→		return strconv.ParseInt(name[:len(name)-5], 10, 64)
   609→	case strings.HasSuffix(name, ".bin"):
   610→		return strconv.ParseInt(name[:len(name)-4], 10, 64)
   611→	default:
   612→		return 0, fmt.Errorf("unknown checkpoint extension for file %q", name)
   613→	}
   614→}
   615→
   616→// findFiles returns filenames from direntries whose timestamps satisfy the filter.
   617→// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the
   618→// last file before t if t falls between two files).
   619→func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {

func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {
	direntries, err := os.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return 0, nil
		}
		return 0, err
	}

	allFiles := make([]fs.DirEntry, 0)
	var walEntry fs.DirEntry
	filesLoaded := 0

	for _, e := range direntries {
		if e.IsDir() {
			// Legacy: skip subdirectories (only used by old Avro format).
			// These are ignored; their data is not loaded.
			cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir)
			continue
		}

		name := e.Name()
		if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") {
			allFiles = append(allFiles, e)
		} else if name == "current.wal" {
			walEntry = e
		}
		// Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes).
	}

	files, err := findFiles(allFiles, from, true)
	if err != nil {
		return filesLoaded, err
	}

	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
		".json": l.loadJSONFile,
		".bin":  l.loadBinaryFile,
	}

	for _, filename := range files {
		ext := filepath.Ext(filename)
		loader := loaders[ext]
		if loader == nil {
			cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename)
			continue
		}

		err := func() error {
			f, err := os.Open(path.Join(dir, filename))
			if err != nil {
				return err
			}
			defer f.Close()
			return loader(m, f, from)
		}()
		if err != nil {
			return filesLoaded, err
		}
		filesLoaded++
	}

	// Replay WAL after all snapshot files so it fills in data since the last snapshot.
	if walEntry != nil {
		err := func() error {
			f, err := os.Open(path.Join(dir, walEntry.Name()))
			if err != nil {
				return err
			}
			defer f.Close()
			return l.loadWALFile(m, f, from)
		}()
		if err != nil {
			// WAL errors are non-fatal: the snapshot already loaded the bulk of data.
			cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err)
		} else {
			filesLoaded++
		}
	}

	return filesLoaded, nil
}

// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename.
// Supports ".json" (format: "<ts>.json") and ".bin" (format: "<ts>.bin").
func parseTimestampFromFilename(name string) (int64, error) {
	switch {
	case strings.HasSuffix(name, ".json"):
		return strconv.ParseInt(name[:len(name)-5], 10, 64)
	case strings.HasSuffix(name, ".bin"):
		return strconv.ParseInt(name[:len(name)-4], 10, 64)
	default:
		return 0, fmt.Errorf("unknown checkpoint extension for file %q", name)
	}
}

// findFiles returns filenames from direntries whose timestamps satisfy the filter.
// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the
// last file before t if t falls between two files).
func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {


Now let me look at `loadBinaryFile` and `loadWALFile`:


pkg/metricstore/walCheckpoint.go:544:func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
pkg/metricstore/walCheckpoint.go:811:func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {


   544→func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
   545→	br := bufio.NewReader(f)
   546→
   547→	// Verify file header magic.
   548→	var fileMagic uint32
   549→	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
   550→		if err == io.EOF {
   551→			return nil // Empty file, no data
   552→		}
   553→		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
   554→	}
   555→
   556→	if fileMagic != walFileMagic {
   557→		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
   558→	}
   559→
   560→	// Cache level lookups to avoid repeated tree traversal.
   561→	lvlCache := make(map[string]*Level)
   562→
   563→	for {
   564→		msg, err := readWALRecord(br)
   565→		if err != nil {
   566→			// Truncated trailing record is expected after a crash; stop replaying.
   567→			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
   568→			break
   569→		}
   570→		if msg == nil {
   571→			break // Clean EOF
   572→		}
   573→
   574→		if msg.Timestamp < from {
   575→			continue // Older than retention window
   576→		}
   577→
   578→		minfo, ok := m.Metrics[msg.MetricName]
   579→		if !ok {
   580→			continue // Unknown metric (config may have changed)
   581→		}
   582→
   583→		// Cache key is the null-separated selector path.
   584→		cacheKey := joinSelector(msg.Selector)
   585→		lvl, ok := lvlCache[cacheKey]
   586→		if !ok {
   587→			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
   588→			lvlCache[cacheKey] = lvl
   589→		}
   590→
   591→		// Write directly to the buffer, same as WriteToLevel but without the
   592→		// global level lookup (we already have the right level).
   593→		lvl.lock.Lock()
   594→		b := lvl.metrics[minfo.offset]
   595→		if b == nil {
   596→			b = newBuffer(msg.Timestamp, minfo.Frequency)
   597→			lvl.metrics[minfo.offset] = b
   598→		}
   599→		nb, writeErr := b.write(msg.Timestamp, msg.Value)
   600→		if writeErr == nil && b != nb {
   601→			lvl.metrics[minfo.offset] = nb
   602→		}
   603→		// Ignore write errors for timestamps before buffer start (can happen when
   604→		// replaying WAL entries that predate a loaded snapshot's start time).
   605→		lvl.lock.Unlock()
   606→	}
   607→
   608→	return nil
   609→}
   610→
   611→// joinSelector builds a cache key from a selector slice using null bytes as separators.
   612→func joinSelector(sel []string) string {
   613→	if len(sel) == 0 {
   614→		return ""
   615→	}
   616→	var result strings.Builder
   617→	result.WriteString(sel[0])
   618→	for i := 1; i < len(sel); i++ {
   619→		result.WriteString("\x00" + sel[i])
   620→	}
   621→	return result.String()
   622→}
   623→
   624→// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
   625→// Returns the number of files written, the list of host directories that were
   626→// successfully checkpointed (for WAL rotation), and any errors.
   627→func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
   628→	// Collect all cluster/host pairs.
   629→	m.root.lock.RLock()
   630→	totalHosts := 0
   631→	for _, l1 := range m.root.children {
   632→		l1.lock.RLock()
   633→		totalHosts += len(l1.children)
   634→		l1.lock.RUnlock()
   635→	}
   636→	m.root.lock.RUnlock()
   637→
   638→	levels := make([]*Level, 0, totalHosts)
   639→	selectors := make([][]string, 0, totalHosts)
   640→
   641→	m.root.lock.RLock()
   642→	for sel1, l1 := range m.root.children {
   643→		l1.lock.RLock()

func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	// Verify file header magic.
	var fileMagic uint32
	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
		if err == io.EOF {
			return nil // Empty file, no data
		}
		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
	}

	if fileMagic != walFileMagic {
		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
	}

	// Cache level lookups to avoid repeated tree traversal.
	lvlCache := make(map[string]*Level)

	for {
		msg, err := readWALRecord(br)
		if err != nil {
			// Truncated trailing record is expected after a crash; stop replaying.
			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
			break
		}
		if msg == nil {
			break // Clean EOF
		}

		if msg.Timestamp < from {
			continue // Older than retention window
		}

		minfo, ok := m.Metrics[msg.MetricName]
		if !ok {
			continue // Unknown metric (config may have changed)
		}

		// Cache key is the null-separated selector path.
		cacheKey := joinSelector(msg.Selector)
		lvl, ok := lvlCache[cacheKey]
		if !ok {
			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
			lvlCache[cacheKey] = lvl
		}

		// Write directly to the buffer, same as WriteToLevel but without the
		// global level lookup (we already have the right level).
		lvl.lock.Lock()
		b := lvl.metrics[minfo.offset]
		if b == nil {
			b = newBuffer(msg.Timestamp, minfo.Frequency)
			lvl.metrics[minfo.offset] = b
		}
		nb, writeErr := b.write(msg.Timestamp, msg.Value)
		if writeErr == nil && b != nb {
			lvl.metrics[minfo.offset] = nb
		}
		// Ignore write errors for timestamps before buffer start (can happen when
		// replaying WAL entries that predate a loaded snapshot's start time).
		lvl.lock.Unlock()
	}

	return nil
}

// joinSelector builds a cache key from a selector slice using null bytes as separators.
func joinSelector(sel []string) string {
	if len(sel) == 0 {
		return ""
	}
	var result strings.Builder
	result.WriteString(sel[0])
	for i := 1; i < len(sel); i++ {
		result.WriteString("\x00" + sel[i])
	}
	return result.String()
}

// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
// Returns the number of files written, the list of host directories that were
// successfully checkpointed (for WAL rotation), and any errors.
func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
	// Collect all cluster/host pairs.
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()


   811→func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
   812→	br := bufio.NewReader(f)
   813→
   814→	var magic uint32
   815→	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
   816→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
   817→	}
   818→	if magic != snapFileMagic {
   819→		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
   820→	}
   821→
   822→	var fileFrom, fileTo int64
   823→	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
   824→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
   825→	}
   826→	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
   827→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
   828→	}
   829→
   830→	if fileTo != 0 && fileTo < from {
   831→		return nil // File is older than retention window, skip it
   832→	}
   833→
   834→	cf, err := readBinaryLevel(br)
   835→	if err != nil {
   836→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
   837→	}
   838→	cf.From = fileFrom
   839→	cf.To = fileTo
   840→
   841→	return l.loadFile(cf, m)
   842→}
   843→
   844→// readBinaryLevel recursively reads a level from the binary snapshot format.
   845→func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
   846→	cf := &CheckpointFile{
   847→		Metrics:  make(map[string]*CheckpointMetrics),
   848→		Children: make(map[string]*CheckpointFile),
   849→	}
   850→
   851→	var numMetrics uint32
   852→	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
   853→		return nil, fmt.Errorf("read num_metrics: %w", err)
   854→	}
   855→
   856→	for range numMetrics {
   857→		name, err := readString16(r)
   858→		if err != nil {
   859→			return nil, fmt.Errorf("read metric name: %w", err)
   860→		}
   861→
   862→		var freq, start int64
   863→		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
   864→			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
   865→		}
   866→		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
   867→			return nil, fmt.Errorf("read start for %s: %w", name, err)
   868→		}
   869→
   870→		var numValues uint32
   871→		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
   872→			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
   873→		}
   874→
   875→		data := make([]schema.Float, numValues)
   876→		for i := range numValues {
   877→			var bits uint32
   878→			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
   879→				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
   880→			}
   881→			data[i] = schema.Float(math.Float32frombits(bits))
   882→		}
   883→
   884→		cf.Metrics[name] = &CheckpointMetrics{
   885→			Frequency: freq,
   886→			Start:     start,
   887→			Data:      data,
   888→		}
   889→	}
   890→
   891→	var numChildren uint32
   892→	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
   893→		return nil, fmt.Errorf("read num_children: %w", err)
   894→	}
   895→
   896→	for range numChildren {
   897→		childName, err := readString16(r)
   898→		if err != nil {
   899→			return nil, fmt.Errorf("read child name: %w", err)
   900→		}
   901→
   902→		child, err := readBinaryLevel(r)
   903→		if err != nil {
   904→			return nil, fmt.Errorf("read child %s: %w", childName, err)
   905→		}
   906→		cf.Children[childName] = child
   907→	}
   908→
   909→	return cf, nil
   910→}

func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	var magic uint32
	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
	}
	if magic != snapFileMagic {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
	}

	var fileFrom, fileTo int64
	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
	}
	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
	}

	if fileTo != 0 && fileTo < from {
		return nil // File is older than retention window, skip it
	}

	cf, err := readBinaryLevel(br)
	if err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
	}
	cf.From = fileFrom
	cf.To = fileTo

	return l.loadFile(cf, m)
}

// readBinaryLevel recursively reads a level from the binary snapshot format.
func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
	cf := &CheckpointFile{
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	var numMetrics uint32
	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
		return nil, fmt.Errorf("read num_metrics: %w", err)
	}

	for range numMetrics {
		name, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read metric name: %w", err)
		}

		var freq, start int64
		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
		}
		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
			return nil, fmt.Errorf("read start for %s: %w", name, err)
		}

		var numValues uint32
		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
		}

		data := make([]schema.Float, numValues)
		for i := range numValues {
			var bits uint32
			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
			}
			data[i] = schema.Float(math.Float32frombits(bits))
		}

		cf.Metrics[name] = &CheckpointMetrics{
			Frequency: freq,
			Start:     start,
			Data:      data,
		}
	}

	var numChildren uint32
	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
		return nil, fmt.Errorf("read num_children: %w", err)
	}

	for range numChildren {
		childName, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read child name: %w", err)
		}

		child, err := readBinaryLevel(r)
		if err != nil {
			return nil, fmt.Errorf("read child %s: %w", childName, err)
		}
		cf.Children[childName] = child
	}

	return cf, nil
}


The `readBinaryLevel` function uses `binary.Read` for every single value - that's extremely slow due to reflection overhead. But let me check if this code actually changed recently or if it was always like this.


git log --oneline --all -- pkg/metricstore/walCheckpoint.go | head -20

Full history of walCheckpoint.go


6f7dda53 Cleanup
0325d9e8 fix: Increase throughput for WAL writers
96a79098 carry forward: uncommitted session files
d3b94c33 Completed 'Explore' agent: Find CUDA/HIP macro switching (toolu_01EbN3)
bf1a8a17 fix: Shard WAL consumer for higher throughput
50aed595 fix: metricstore NATS contention
818a05f2 carry forward: uncommitted session files
b214e175 Add buffered I/O to WAL writes and fix MemoryCap comment
ecd29b28 carry forward: uncommitted session files
21b6c1b5 carry forward: uncommitted session files
a243e174 Update to shutdown worker for WAL checkpointing mode
a418abc7 Run go fix
ca0f9a42 Introduce metric store binary checkpoints with write ahead log


Check the "Increase throughput" commit


commit 0325d9e866ec97994e77770ca189838c0ee3b3f0
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Tue Mar 24 06:53:12 2026 +0100

    fix: Increase throughput for WAL writers

    Entire-Checkpoint: ddd40d290c56

 pkg/metricstore/walCheckpoint.go | 63 ++++++++++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 21 deletions(-)


Check the "Cleanup" commit


commit 6f7dda53ee1d98481855effeadfd8a2e7d26c7ec
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Tue Mar 24 07:03:46 2026 +0100

    Cleanup

    Entire-Checkpoint: ed68d32218ac

 pkg/metricstore/walCheckpoint.go | 63 ----------------------------------------
 1 file changed, 63 deletions(-)


git diff 6f7dda53^..6f7dda53 -- pkg/metricstore/walCheckpoint.go

Show the Cleanup commit diff


diff --git a/pkg/metricstore/walCheckpoint.go b/pkg/metricstore/walCheckpoint.go
index 85e5047e..57064e5f 100644
--- a/pkg/metricstore/walCheckpoint.go
+++ b/pkg/metricstore/walCheckpoint.go
@@ -434,69 +434,6 @@ func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
 	return err
 }

-// buildWALPayload encodes a WALMessage into a binary payload (without magic/length/CRC).
-func buildWALPayload(msg *WALMessage) []byte {
-	size := 8 + 2 + len(msg.MetricName) + 1 + 4
-	for _, s := range msg.Selector {
-		size += 1 + len(s)
-	}
-
-	buf := make([]byte, 0, size)
-
-	// Timestamp (8 bytes, little-endian int64)
-	var ts [8]byte
-	binary.LittleEndian.PutUint64(ts[:], uint64(msg.Timestamp))
-	buf = append(buf, ts[:]...)
-
-	// Metric name (2-byte length prefix + bytes)
-	var mLen [2]byte
-	binary.LittleEndian.PutUint16(mLen[:], uint16(len(msg.MetricName)))
-	buf = append(buf, mLen[:]...)
-	buf = append(buf, msg.MetricName...)
-
-	// Selector count (1 byte)
-	buf = append(buf, byte(len(msg.Selector)))
-
-	// Selectors (1-byte length prefix + bytes each)
-	for _, sel := range msg.Selector {
-		buf = append(buf, byte(len(sel)))
-		buf = append(buf, sel...)
-	}
-
-	// Value (4 bytes, float32 bit representation)
-	var val [4]byte
-	binary.LittleEndian.PutUint32(val[:], math.Float32bits(float32(msg.Value)))
-	buf = append(buf, val[:]...)
-
-	return buf
-}
-
-// writeWALRecord appends a binary WAL record to the writer.
-// Format: [4B magic][4B payload_len][payload][4B CRC32]
-func writeWALRecord(w io.Writer, msg *WALMessage) error {
-	payload := buildWALPayload(msg)
-	crc := crc32.ChecksumIEEE(payload)
-
-	record := make([]byte, 0, 4+4+len(payload)+4)
-
-	var magic [4]byte
-	binary.LittleEndian.PutUint32(magic[:], walRecordMagic)
-	record = append(record, magic[:]...)
-
-	var pLen [4]byte
-	binary.LittleEndian.PutUint32(pLen[:], uint32(len(payload)))
-	record = append(record, pLen[:]...)
-
-	record = append(record, payload...)
-
-	var crcBytes [4]byte
-	binary.LittleEndian.PutUint32(crcBytes[:], crc)
-	record = append(record, crcBytes[:]...)
-
-	_, err := w.Write(record)
-	return err
-}
-
 // readWALRecord reads one WAL record from the reader.
 // Returns (nil, nil) on clean EOF. Returns error on data corruption.
 // A CRC mismatch indicates a truncated trailing record (expected on crash).


The recent commits only affected writing, not loading. The loading code hasn't changed. So the regression must be because the checkpoint format is now WAL+binary (`.bin` files) instead of JSON, and `readBinaryLevel` uses `binary.Read` per value which is extremely slow due to reflection.

Let me check what format was being loaded before:


git log --oneline --all -- pkg/metricstore/checkpoint.go pkg/metricstore/walCheckpoint.go | head -20

Timeline of checkpoint changes


6f7dda53 Cleanup
0325d9e8 fix: Increase throughput for WAL writers
96a79098 carry forward: uncommitted session files
d3b94c33 Completed 'Explore' agent: Find CUDA/HIP macro switching (toolu_01EbN3)
bf1a8a17 fix: Shard WAL consumer for higher throughput
50aed595 fix: metricstore NATS contention
818a05f2 carry forward: uncommitted session files
39ab1278 Make checkpointInterval an option config option again.
b214e175 Add buffered I/O to WAL writes and fix MemoryCap comment
ecd29b28 carry forward: uncommitted session files
21b6c1b5 carry forward: uncommitted session files
39635ea1 Cleanup metricstore options
a243e174 Update to shutdown worker for WAL checkpointing mode
a418abc7 Run go fix
ca0f9a42 Introduce metric store binary checkpoints with write ahead log
9fc1836c Replace avro checkpoints with native binary format
8b7cb587 Modernize code
41546796 Apply optimizations to checkpoint loading
3cf88f75 Update to checkpoint loader in CCMS
eb5aa9ad Disable explicit GC calls


git log --format="%h %ai %s" 93a9d732..HEAD -- pkg/metricstore/

Commits between working and broken state


(Bash completed with no output)


git log --format="%h %ai %s" --all -- pkg/metricstore/ | head -30

Timeline with dates


6f7dda53 2026-03-24 07:03:46 +0100 Cleanup
0325d9e8 2026-03-24 06:53:12 +0100 fix: Increase throughput for WAL writers
e41d1251 2026-03-23 06:37:24 +0100 fix: Continue on error
586c9020 2026-03-23 06:32:24 +0100 Restructure metricstore cleanup archiving to stay withinh 32k parquet-go limit
96a79098 2026-03-20 08:03:34 +0100 carry forward: uncommitted session files
c0d2d65f 2026-03-19 21:16:48 +0100 Introduce average resampler support
09501df3 2026-03-18 17:32:16 +0100 fix: reduce memory usage in parquet checkpoint archiver
d3b94c33 2026-03-18 15:48:25 +0100 Completed 'Explore' agent: Find CUDA/HIP macro switching (toolu_01EbN3)
8b132ed7 2026-03-18 06:47:45 +0100 fix: Blocking ReceiveNats call
bf1a8a17 2026-03-18 06:32:14 +0100 fix: Shard WAL consumer for higher throughput
50aed595 2026-03-18 06:14:15 +0100 fix: metricstore NATS contention
818a05f2 2026-03-18 05:31:49 +0100 carry forward: uncommitted session files
d46e6371 2026-03-18 05:22:39 +0100 Add log about checkpoint archiving
02f82c2c 2026-03-18 05:08:37 +0100 fix: Prevent memory spikes in parquet writer for  metricstore move policy
0a4c4d8e 2026-03-17 18:41:47 +0100 Patching parquet archive writer for high memory usage
bab6eb4c 2026-03-16 15:35:24 +0100 Convert Warn message on missing metrics to Debug level
39ab1278 2026-03-13 09:07:38 +0100 Make checkpointInterval an option config option again.
b214e175 2026-03-13 09:05:24 +0100 Add buffered I/O to WAL writes and fix MemoryCap comment
8234ad31 2026-03-13 07:57:35 +0100 fix: Fix metricstore memory explosion from broken emergency free and batch aborts
ecd29b28 2026-03-04 17:31:37 +0100 carry forward: uncommitted session files
845d0111 2026-03-04 17:31:36 +0100 Further consolidate and improve ccms query builder
26982088 2026-03-04 16:43:05 +0100 Consolidate code for external and internal ccms buildQueries function
67a17b53 2026-03-04 15:14:35 +0100 Reduce noise in info log
21b6c1b5 2026-03-04 10:38:10 +0100 carry forward: uncommitted session files
39635ea1 2026-03-04 10:37:43 +0100 Cleanup metricstore options
74ab51f4 2026-03-03 09:51:04 +0100 Patch bufferPool with no limits to pool size
688ad507 2026-03-03 06:58:28 +0100 Merge branch 'optimize-checkpoint-wal' into dev
718ff602 2026-03-02 14:10:28 +0100 clarify ccms logs
a243e174 2026-03-02 15:27:06 +0100 Update to shutdown worker for WAL checkpointing mode
1ec41d83 2026-02-28 19:34:33 +0100 Review and improve buffer pool implmentation. Add unit tests.


Let me look at `readWALRecord` to understand WAL replay performance:


440:func readWALRecord(r io.Reader) (*WALMessage, error) {


   434→	return err
   435→}
   436→
   437→// readWALRecord reads one WAL record from the reader.
   438→// Returns (nil, nil) on clean EOF. Returns error on data corruption.
   439→// A CRC mismatch indicates a truncated trailing record (expected on crash).
   440→func readWALRecord(r io.Reader) (*WALMessage, error) {
   441→	var magic uint32
   442→	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
   443→		if err == io.EOF {
   444→			return nil, nil // Clean EOF
   445→		}
   446→		return nil, fmt.Errorf("read record magic: %w", err)
   447→	}
   448→
   449→	if magic != walRecordMagic {
   450→		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
   451→	}
   452→
   453→	var payloadLen uint32
   454→	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
   455→		return nil, fmt.Errorf("read payload length: %w", err)
   456→	}
   457→
   458→	if payloadLen > 1<<20 { // 1 MB sanity limit
   459→		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
   460→	}
   461→
   462→	payload := make([]byte, payloadLen)
   463→	if _, err := io.ReadFull(r, payload); err != nil {
   464→		return nil, fmt.Errorf("read payload: %w", err)
   465→	}
   466→
   467→	var storedCRC uint32
   468→	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
   469→		return nil, fmt.Errorf("read CRC: %w", err)
   470→	}
   471→
   472→	if crc32.ChecksumIEEE(payload) != storedCRC {
   473→		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
   474→	}
   475→
   476→	return parseWALPayload(payload)
   477→}
   478→
   479→// parseWALPayload decodes a binary payload into a WALMessage.
   480→func parseWALPayload(payload []byte) (*WALMessage, error) {
   481→	if len(payload) < 8+2+1+4 {
   482→		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
   483→	}
   484→
   485→	offset := 0
   486→
   487→	// Timestamp (8 bytes)
   488→	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
   489→	offset += 8
   490→
   491→	// Metric name (2-byte length + bytes)
   492→	if offset+2 > len(payload) {
   493→		return nil, fmt.Errorf("metric name length overflows payload")
   494→	}
   495→	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
   496→	offset += 2
   497→
   498→	if offset+mLen > len(payload) {
   499→		return nil, fmt.Errorf("metric name overflows payload")
   500→	}
   501→	metricName := string(payload[offset : offset+mLen])
   502→	offset += mLen
   503→
   504→	// Selector count (1 byte)
   505→	if offset >= len(payload) {
   506→		return nil, fmt.Errorf("selector count overflows payload")
   507→	}
   508→	selCount := int(payload[offset])
   509→	offset++
   510→
   511→	selectors := make([]string, selCount)
   512→	for i := range selCount {
   513→		if offset >= len(payload) {
   514→			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
   515→		}
   516→		sLen := int(payload[offset])
   517→		offset++
   518→
   519→		if offset+sLen > len(payload) {
   520→			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
   521→		}
   522→		selectors[i] = string(payload[offset : offset+sLen])
   523→		offset += sLen
   524→	}
   525→
   526→	// Value (4 bytes, float32 bits)
   527→	if offset+4 > len(payload) {
   528→		return nil, fmt.Errorf("value overflows payload")
   529→	}
   530→	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
   531→	value := schema.Float(math.Float32frombits(bits))
   532→
   533→	return &WALMessage{
   534→		MetricName: metricName,
   535→		Timestamp:  ts,
   536→		Selector:   selectors,
   537→		Value:      value,
   538→	}, nil
   539→}
   540→
   541→// loadWALFile reads a WAL file and replays all valid records into the Level tree.
   542→// l is the host-level node. Corrupt or partial trailing records are silently skipped
   543→// (expected on crash). Records older than 'from' are skipped.

	return err
}

// readWALRecord reads one WAL record from the reader.
// Returns (nil, nil) on clean EOF. Returns error on data corruption.
// A CRC mismatch indicates a truncated trailing record (expected on crash).
func readWALRecord(r io.Reader) (*WALMessage, error) {
	var magic uint32
	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
		if err == io.EOF {
			return nil, nil // Clean EOF
		}
		return nil, fmt.Errorf("read record magic: %w", err)
	}

	if magic != walRecordMagic {
		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
	}

	var payloadLen uint32
	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
		return nil, fmt.Errorf("read payload length: %w", err)
	}

	if payloadLen > 1<<20 { // 1 MB sanity limit
		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
	}

	payload := make([]byte, payloadLen)
	if _, err := io.ReadFull(r, payload); err != nil {
		return nil, fmt.Errorf("read payload: %w", err)
	}

	var storedCRC uint32
	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
		return nil, fmt.Errorf("read CRC: %w", err)
	}

	if crc32.ChecksumIEEE(payload) != storedCRC {
		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
	}

	return parseWALPayload(payload)
}

// parseWALPayload decodes a binary payload into a WALMessage.
func parseWALPayload(payload []byte) (*WALMessage, error) {
	if len(payload) < 8+2+1+4 {
		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
	}

	offset := 0

	// Timestamp (8 bytes)
	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
	offset += 8

	// Metric name (2-byte length + bytes)
	if offset+2 > len(payload) {
		return nil, fmt.Errorf("metric name length overflows payload")
	}
	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
	offset += 2

	if offset+mLen > len(payload) {
		return nil, fmt.Errorf("metric name overflows payload")
	}
	metricName := string(payload[offset : offset+mLen])
	offset += mLen

	// Selector count (1 byte)
	if offset >= len(payload) {
		return nil, fmt.Errorf("selector count overflows payload")
	}
	selCount := int(payload[offset])
	offset++

	selectors := make([]string, selCount)
	for i := range selCount {
		if offset >= len(payload) {
			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
		}
		sLen := int(payload[offset])
		offset++

		if offset+sLen > len(payload) {
			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
		}
		selectors[i] = string(payload[offset : offset+sLen])
		offset += sLen
	}

	// Value (4 bytes, float32 bits)
	if offset+4 > len(payload) {
		return nil, fmt.Errorf("value overflows payload")
	}
	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
	value := schema.Float(math.Float32frombits(bits))

	return &WALMessage{
		MetricName: metricName,
		Timestamp:  ts,
		Selector:   selectors,
		Value:      value,
	}, nil
}

// loadWALFile reads a WAL file and replays all valid records into the Level tree.
// l is the host-level node. Corrupt or partial trailing records are silently skipped
// (expected on crash). Records older than 'from' are skipped.


Let me check the worker pool size and the `readBinaryLevel` performance characteristics:


pkg/metricstore/metricstore.go:136:	// Set NumWorkers from config or use default
pkg/metricstore/metricstore.go:137:	if Keys.NumWorkers <= 0 {
pkg/metricstore/metricstore.go:138:		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
pkg/metricstore/metricstore.go:140:	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
pkg/metricstore/metricstore.go:188:			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
pkg/metricstore/config.go:13://	├─ NumWorkers: Parallel checkpoint/archive workers
pkg/metricstore/config.go:114://   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
pkg/metricstore/config.go:124:	NumWorkers         int            `json:"num-workers"`
pkg/metricstore/checkpoint.go:184:// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
pkg/metricstore/checkpoint.go:222:	wg.Add(Keys.NumWorkers)
pkg/metricstore/checkpoint.go:223:	work := make(chan workItem, Keys.NumWorkers*2)
pkg/metricstore/checkpoint.go:224:	for worker := 0; worker < Keys.NumWorkers; worker++ {
pkg/metricstore/checkpoint.go:397:	work := make(chan [2]string, Keys.NumWorkers*4)
pkg/metricstore/checkpoint.go:400:	wg.Add(Keys.NumWorkers)
pkg/metricstore/checkpoint.go:401:	for worker := 0; worker < Keys.NumWorkers; worker++ {
pkg/metricstore/archive.go:109:	work := make(chan workItem, Keys.NumWorkers)
pkg/metricstore/archive.go:111:	wg.Add(Keys.NumWorkers)
pkg/metricstore/archive.go:112:	for worker := 0; worker < Keys.NumWorkers; worker++ {
pkg/metricstore/archive.go:208:		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
pkg/metricstore/archive.go:212:		}, Keys.NumWorkers)
pkg/metricstore/archive.go:217:		wg.Add(Keys.NumWorkers)
pkg/metricstore/archive.go:218:		for w := 0; w < Keys.NumWorkers; w++ {
pkg/metricstore/walCheckpoint.go:159:	walNumShards = max(Keys.NumWorkers, 1)
pkg/metricstore/walCheckpoint.go:663:	wg.Add(Keys.NumWorkers)
pkg/metricstore/walCheckpoint.go:664:	work := make(chan workItem, Keys.NumWorkers*2)
pkg/metricstore/walCheckpoint.go:666:	for range Keys.NumWorkers {


Now let me look at `readBinaryLevel` more carefully — specifically the float reading loop and `readString16`:


913:func readString16(r io.Reader) (string, error) {
914-	var sLen uint16
915-	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
916-		return "", err
917-	}
918-	buf := make([]byte, sLen)
919-	if _, err := io.ReadFull(r, buf); err != nil {
920-		return "", err
921-	}
922-	return string(buf), nil
923-}


[Request interrupted by user]


It must have been changes of the last 2 days. Before the startup was fast. Maybe the changes in the startup and shutdown handling code?


Good point. Let me look at the shutdown/startup changes from the last 2 days.


git log --oneline --since="2026-03-23" --all

All commits from last 2 days


4b6f6941 Checkpoint: a4d012e1edcf
93a9d732 fix: Improve shutdown time
889b284f Checkpoint: ed68d32218ac
6f7dda53 Cleanup
bc9c47cd Checkpoint: ddd40d290c56
0325d9e8 fix: Increase throughput for WAL writers
c13fd68a Merge pull request #534 from ClusterCockpit/hotfix
3d94b0bf Merge branch 'master' into hotfix
d5ea2b4c change: query node states explicitly in node view
7c42fc3a Checkpoint: 29b9d52db89c
45f329e5 feat: Add command line switch to trigger manual metricstore checkpoint cleanup
100dd7da Merge pull request #533 from ClusterCockpit/hotfix
493cee05 Checkpoint: ccda3b2ff4cb
192c94a7 fix: Prevent interruption of body lineprotocol parsing on locks
40142966 Checkpoint: 6000eb5a5bb8
e41d1251 fix: Continue on error
1abea621 Checkpoint: 1660b8cf2571
586c9020 Restructure metricstore cleanup archiving to stay withinh 32k parquet-go limit
38f75ceb Checkpoint: 1660b8cf2571


Check "Improve shutdown time" commit


commit 93a9d732a48cf3c9dc1969d467c8a4b6ecf99adb
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Tue Mar 24 07:17:34 2026 +0100

    fix: Improve shutdown time

    Entire-Checkpoint: a4d012e1edcf

 cmd/cc-backend/server.go | 48 +++++++++++++++++++++++++++---------------------
 1 file changed, 27 insertions(+), 21 deletions(-)


Check manual checkpoint cleanup commit


commit 45f329e5fbdf0643246f3413b9831523bde7d2c3
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Mon Mar 23 07:58:35 2026 +0100

    feat: Add command line switch to trigger manual metricstore checkpoint cleanup

    Entire-Checkpoint: 29b9d52db89c

 cmd/cc-backend/cli.go  |  4 +++-
 cmd/cc-backend/main.go | 38 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)


git diff 93a9d732^..93a9d732 -- cmd/cc-backend/server.go

Show the shutdown time improvement diff


diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go
index 4e6120e9..b37d3450 100644
--- a/cmd/cc-backend/server.go
+++ b/cmd/cc-backend/server.go
@@ -18,6 +18,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync"
 	"time"

 	"github.com/99designs/gqlgen/graphql"
@@ -399,16 +400,6 @@ func (s *Server) Start(ctx context.Context) error {
 		return fmt.Errorf("dropping privileges: %w", err)
 	}

-	// Handle context cancellation for graceful shutdown
-	go func() {
-		<-ctx.Done()
-		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-		defer cancel()
-		if err := s.server.Shutdown(shutdownCtx); err != nil {
-			cclog.Errorf("Server shutdown error: %v", err)
-		}
-	}()
-
 	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
 		return fmt.Errorf("server failed: %w", err)
 	}
@@ -416,8 +407,7 @@ func (s *Server) Start(ctx context.Context) error {
 }

 func (s *Server) Shutdown(ctx context.Context) {
-	// Create a shutdown context with timeout
-	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
 	defer cancel()

 	nc := nats.GetClient()
@@ -425,20 +415,36 @@ func (s *Server) Shutdown(ctx context.Context) {
 		nc.Close()
 	}

-	// First shut down the server gracefully (waiting for all ongoing requests)
 	if err := s.server.Shutdown(shutdownCtx); err != nil {
 		cclog.Errorf("Server shutdown error: %v", err)
 	}

-	// Archive all the metric store data
-	ms := metricstore.GetMemoryStore()
+	// Run metricstore and archiver shutdown concurrently.
+	// They are independent: metricstore writes .bin snapshots,
+	// archiver flushes pending job archives.
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		var wg sync.WaitGroup

-	if ms != nil {
-		metricstore.Shutdown()
-	}
+		if ms := metricstore.GetMemoryStore(); ms != nil {
+			wg.Go(func() {
+				metricstore.Shutdown()
+			})
+		}
+
+		wg.Go(func() {
+			if err := archiver.Shutdown(10 * time.Second); err != nil {
+				cclog.Warnf("Archiver shutdown: %v", err)
+			}
+		})
+
+		wg.Wait()
+	}()

-	// Shutdown archiver with 10 second timeout for fast shutdown
-	if err := archiver.Shutdown(10 * time.Second); err != nil {
-		cclog.Warnf("Archiver shutdown: %v", err)
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		cclog.Warn("Shutdown deadline exceeded, forcing exit")
 	}
 }


git diff 45f329e5^..45f329e5 -- cmd/cc-backend/main.go cmd/cc-backend/cli.go

Show manual checkpoint cleanup commit diff


diff --git a/cmd/cc-backend/cli.go b/cmd/cc-backend/cli.go
index 47f534be..3896383a 100644
--- a/cmd/cc-backend/cli.go
+++ b/cmd/cc-backend/cli.go
@@ -11,7 +11,8 @@ import "flag"

 var (
 	flagReinitDB, flagInit, flagServer, flagSyncLDAP, flagGops, flagMigrateDB, flagRevertDB,
-	flagForceDB, flagDev, flagVersion, flagLogDateTime, flagApplyTags, flagOptimizeDB bool
+	flagForceDB, flagDev, flagVersion, flagLogDateTime, flagApplyTags, flagOptimizeDB,
+	flagCleanupCheckpoints bool
 	flagNewUser, flagDelUser, flagGenJWT, flagConfigFile, flagImportJob, flagLogLevel string
 )

@@ -28,6 +29,7 @@ func cliInit() {
 	flag.BoolVar(&flagApplyTags, "apply-tags", false, "Run taggers on all completed jobs and exit")
 	flag.BoolVar(&flagForceDB, "force-db", false, "Force database version, clear dirty flag and exit")
 	flag.BoolVar(&flagOptimizeDB, "optimize-db", false, "Optimize database: run VACUUM to reclaim space, then ANALYZE to update query planner statistics")
+	flag.BoolVar(&flagCleanupCheckpoints, "cleanup-checkpoints", false, "Clean up old checkpoint files (delete or archive) based on retention settings, then exit")
 	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
 	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
 	flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: <username>:[admin,support,manager,api,user]:<password>")
diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go
index 12faeae5..f8f0a767 100644
--- a/cmd/cc-backend/main.go
+++ b/cmd/cc-backend/main.go
@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	goruntime "runtime"
 	"runtime/debug"
 	"strings"
 	"sync"
@@ -536,6 +537,43 @@ func run() error {
 		return err
 	}

+	// Handle checkpoint cleanup
+	if flagCleanupCheckpoints {
+		mscfg := ccconf.GetPackageConfig("metric-store")
+		if mscfg == nil {
+			return fmt.Errorf("metric-store configuration required for checkpoint cleanup")
+		}
+		if err := json.Unmarshal(mscfg, &metricstore.Keys); err != nil {
+			return fmt.Errorf("decoding metric-store config: %w", err)
+		}
+		if metricstore.Keys.NumWorkers <= 0 {
+			metricstore.Keys.NumWorkers = min(goruntime.NumCPU()/2+1, metricstore.DefaultMaxWorkers)
+		}
+
+		d, err := time.ParseDuration(metricstore.Keys.RetentionInMemory)
+		if err != nil {
+			return fmt.Errorf("parsing retention-in-memory: %w", err)
+		}
+		from := time.Now().Add(-d)
+		deleteMode := metricstore.Keys.Cleanup == nil || metricstore.Keys.Cleanup.Mode != "archive"
+		cleanupDir := ""
+		if !deleteMode {
+			cleanupDir = metricstore.Keys.Cleanup.RootDir
+		}
+
+		cclog.Infof("Cleaning up checkpoints older than %s...", from.Format(time.RFC3339))
+		n, err := metricstore.CleanupCheckpoints(
+			metricstore.Keys.Checkpoints.RootDir, cleanupDir, from.Unix(), deleteMode)
+		if err != nil {
+			return fmt.Errorf("checkpoint cleanup: %w", err)
+		}
+		if deleteMode {
+			cclog.Exitf("Cleanup done: %d checkpoint files deleted.", n)
+		} else {
+			cclog.Exitf("Cleanup done: %d checkpoint files archived to parquet.", n)
+		}
+	}
+
 	// Exit if start server is not requested
 	if !flagServer {
 		cclog.Exit("No errors, server flag not set. Exiting cc-backend.")


Show the lineprotocol parsing fix


diff --git a/cmd/cc-backend/server.go b/cmd/cc-backend/server.go
index bdbb8a7e..4e6120e9 100644
--- a/cmd/cc-backend/server.go
+++ b/cmd/cc-backend/server.go
@@ -344,18 +344,18 @@ func (s *Server) init() error {

 // Server timeout defaults (in seconds)
 const (
-	defaultReadTimeout  = 20
-	defaultWriteTimeout = 20
+	defaultReadHeaderTimeout = 20
+	defaultWriteTimeout      = 20
 )

 func (s *Server) Start(ctx context.Context) error {
 	// Use configurable timeouts with defaults
-	readTimeout := time.Duration(defaultReadTimeout) * time.Second
+	readHeaderTimeout := time.Duration(defaultReadHeaderTimeout) * time.Second
 	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second

 	s.server = &http.Server{
-		ReadTimeout:  readTimeout,
-		WriteTimeout: writeTimeout,
+		ReadHeaderTimeout: readHeaderTimeout,
+		WriteTimeout:      writeTimeout,
 		Handler:      s.router,
 		Addr:         config.Keys.Addr,
 	}


   537→		return err
   538→	}
   539→
   540→	// Handle checkpoint cleanup
   541→	if flagCleanupCheckpoints {
   542→		mscfg := ccconf.GetPackageConfig("metric-store")
   543→		if mscfg == nil {
   544→			return fmt.Errorf("metric-store configuration required for checkpoint cleanup")
   545→		}
   546→		if err := json.Unmarshal(mscfg, &metricstore.Keys); err != nil {
   547→			return fmt.Errorf("decoding metric-store config: %w", err)
   548→		}
   549→		if metricstore.Keys.NumWorkers <= 0 {
   550→			metricstore.Keys.NumWorkers = min(goruntime.NumCPU()/2+1, metricstore.DefaultMaxWorkers)
   551→		}
   552→
   553→		d, err := time.ParseDuration(metricstore.Keys.RetentionInMemory)
   554→		if err != nil {
   555→			return fmt.Errorf("parsing retention-in-memory: %w", err)
   556→		}
   557→		from := time.Now().Add(-d)
   558→		deleteMode := metricstore.Keys.Cleanup == nil || metricstore.Keys.Cleanup.Mode != "archive"
   559→		cleanupDir := ""
   560→		if !deleteMode {
   561→			cleanupDir = metricstore.Keys.Cleanup.RootDir
   562→		}
   563→
   564→		cclog.Infof("Cleaning up checkpoints older than %s...", from.Format(time.RFC3339))
   565→		n, err := metricstore.CleanupCheckpoints(
   566→			metricstore.Keys.Checkpoints.RootDir, cleanupDir, from.Unix(), deleteMode)
   567→		if err != nil {
   568→			return fmt.Errorf("checkpoint cleanup: %w", err)
   569→		}
   570→		if deleteMode {
   571→			cclog.Exitf("Cleanup done: %d checkpoint files deleted.", n)
   572→		} else {
   573→			cclog.Exitf("Cleanup done: %d checkpoint files archived to parquet.", n)
   574→		}
   575→	}
   576→
   577→	// Exit if start server is not requested
   578→	if !flagServer {
   579→		cclog.Exit("No errors, server flag not set. Exiting cc-backend.")
   580→	}
   581→
   582→	// Run server with context
   583→	ctx, cancel := context.WithCancel(context.Background())
   584→	defer cancel()
   585→
   586→	return runServer(ctx)
   587→}
   588→
   589→func main() {
   590→	if err := run(); err != nil {
   591→		cclog.Error(err.Error())
   592→		os.Exit(1)
   593→	}
   594→}
   595→

		return err
	}

	// Handle checkpoint cleanup
	if flagCleanupCheckpoints {
		mscfg := ccconf.GetPackageConfig("metric-store")
		if mscfg == nil {
			return fmt.Errorf("metric-store configuration required for checkpoint cleanup")
		}
		if err := json.Unmarshal(mscfg, &metricstore.Keys); err != nil {
			return fmt.Errorf("decoding metric-store config: %w", err)
		}
		if metricstore.Keys.NumWorkers <= 0 {
			metricstore.Keys.NumWorkers = min(goruntime.NumCPU()/2+1, metricstore.DefaultMaxWorkers)
		}

		d, err := time.ParseDuration(metricstore.Keys.RetentionInMemory)
		if err != nil {
			return fmt.Errorf("parsing retention-in-memory: %w", err)
		}
		from := time.Now().Add(-d)
		deleteMode := metricstore.Keys.Cleanup == nil || metricstore.Keys.Cleanup.Mode != "archive"
		cleanupDir := ""
		if !deleteMode {
			cleanupDir = metricstore.Keys.Cleanup.RootDir
		}

		cclog.Infof("Cleaning up checkpoints older than %s...", from.Format(time.RFC3339))
		n, err := metricstore.CleanupCheckpoints(
			metricstore.Keys.Checkpoints.RootDir, cleanupDir, from.Unix(), deleteMode)
		if err != nil {
			return fmt.Errorf("checkpoint cleanup: %w", err)
		}
		if deleteMode {
			cclog.Exitf("Cleanup done: %d checkpoint files deleted.", n)
		} else {
			cclog.Exitf("Cleanup done: %d checkpoint files archived to parquet.", n)
		}
	}

	// Exit if start server is not requested
	if !flagServer {
		cclog.Exit("No errors, server flag not set. Exiting cc-backend.")
	}

	// Run server with context
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	return runServer(ctx)
}

func main() {
	if err := run(); err != nil {
		cclog.Error(err.Error())
		os.Exit(1)
	}
}


Let me look at the `runServer` and the metricstore `Init` call path:


cmd/cc-backend/main.go:339:func runServer(ctx context.Context) error {


   339→func runServer(ctx context.Context) error {
   340→	var wg sync.WaitGroup
   341→
   342→	// Initialize metric store if configuration is provided
   343→	haveMetricstore := false
   344→	mscfg := ccconf.GetPackageConfig("metric-store")
   345→	if mscfg != nil {
   346→		metrics := metricstore.BuildMetricList()
   347→		metricstore.Init(mscfg, metrics, &wg)
   348→
   349→		// Inject repository as NodeProvider to break import cycle
   350→		ms := metricstore.GetMemoryStore()
   351→		jobRepo := repository.GetJobRepository()
   352→		ms.SetNodeProvider(jobRepo)
   353→		metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
   354→		haveMetricstore = true
   355→	} else {
   356→		metricstore.MetricStoreHandle = nil
   357→		cclog.Debug("missing internal metricstore configuration")
   358→	}
   359→
   360→	// Initialize external metric stores if configuration is provided
   361→	mscfg = ccconf.GetPackageConfig("metric-store-external")
   362→	if mscfg != nil {
   363→		err := metricdispatch.Init(mscfg)
   364→
   365→		if err != nil {
   366→			cclog.Debugf("error while initializing external metricdispatch: %v", err)
   367→		} else {
   368→			haveMetricstore = true
   369→		}
   370→	}
   371→
   372→	if !haveMetricstore {
   373→		return fmt.Errorf("missing metricstore configuration")
   374→	}
   375→
   376→	// Start archiver and task manager
   377→	archiver.Start(repository.GetJobRepository(), ctx)
   378→	taskmanager.Start(ccconf.GetPackageConfig("cron"), ccconf.GetPackageConfig("archive"))
   379→
   380→	// Initialize web UI
   381→	cfg := ccconf.GetPackageConfig("ui")
   382→	if err := web.Init(cfg); err != nil {
   383→		return fmt.Errorf("initializing web UI: %w", err)
   384→	}
   385→
   386→	// Initialize HTTP server
   387→	srv, err := NewServer(version, commit, date)
   388→	if err != nil {
   389→		return fmt.Errorf("creating server: %w", err)
   390→	}
   391→
   392→	// Channel to collect errors from server
   393→	errChan := make(chan error, 1)
   394→
   395→	// Start HTTP server
   396→	wg.Go(func() {
   397→		if err := srv.Start(ctx); err != nil {
   398→			errChan <- err
   399→		}
   400→	})
   401→
   402→	// Handle shutdown signals
   403→	wg.Add(1)
   404→	sigs := make(chan os.Signal, 1)
   405→	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
   406→	go func() {
   407→		defer wg.Done()
   408→		select {
   409→		case <-sigs:
   410→			cclog.Info("Shutdown signal received")
   411→		case <-ctx.Done():
   412→		}
   413→
   414→		runtime.SystemdNotify(false, "Shutting down ...")
   415→		srv.Shutdown(ctx)
   416→		util.FsWatcherShutdown()
   417→		taskmanager.Shutdown()
   418→	}()
   419→
   420→	// Set GC percent if not configured
   421→	if os.Getenv(envGOGC) == "" {
   422→		// trigger GC when heap grows 15% above the previous live set
   423→		debug.SetGCPercent(15)
   424→	}
   425→	runtime.SystemdNotify(true, "running")
   426→
   427→	waitDone := make(chan struct{})
   428→	go func() {
   429→		wg.Wait()
   430→		close(waitDone)
   431→	}()
   432→
   433→	go func() {
   434→		<-waitDone
   435→		close(errChan)
   436→	}()
   437→
   438→	// Wait for either:

func runServer(ctx context.Context) error {
	var wg sync.WaitGroup

	// Initialize metric store if configuration is provided
	haveMetricstore := false
	mscfg := ccconf.GetPackageConfig("metric-store")
	if mscfg != nil {
		metrics := metricstore.BuildMetricList()
		metricstore.Init(mscfg, metrics, &wg)

		// Inject repository as NodeProvider to break import cycle
		ms := metricstore.GetMemoryStore()
		jobRepo := repository.GetJobRepository()
		ms.SetNodeProvider(jobRepo)
		metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
		haveMetricstore = true
	} else {
		metricstore.MetricStoreHandle = nil
		cclog.Debug("missing internal metricstore configuration")
	}

	// Initialize external metric stores if configuration is provided
	mscfg = ccconf.GetPackageConfig("metric-store-external")
	if mscfg != nil {
		err := metricdispatch.Init(mscfg)

		if err != nil {
			cclog.Debugf("error while initializing external metricdispatch: %v", err)
		} else {
			haveMetricstore = true
		}
	}

	if !haveMetricstore {
		return fmt.Errorf("missing metricstore configuration")
	}

	// Start archiver and task manager
	archiver.Start(repository.GetJobRepository(), ctx)
	taskmanager.Start(ccconf.GetPackageConfig("cron"), ccconf.GetPackageConfig("archive"))

	// Initialize web UI
	cfg := ccconf.GetPackageConfig("ui")
	if err := web.Init(cfg); err != nil {
		return fmt.Errorf("initializing web UI: %w", err)
	}

	// Initialize HTTP server
	srv, err := NewServer(version, commit, date)
	if err != nil {
		return fmt.Errorf("creating server: %w", err)
	}

	// Channel to collect errors from server
	errChan := make(chan error, 1)

	// Start HTTP server
	wg.Go(func() {
		if err := srv.Start(ctx); err != nil {
			errChan <- err
		}
	})

	// Handle shutdown signals
	wg.Add(1)
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		defer wg.Done()
		select {
		case <-sigs:
			cclog.Info("Shutdown signal received")
		case <-ctx.Done():
		}

		runtime.SystemdNotify(false, "Shutting down ...")
		srv.Shutdown(ctx)
		util.FsWatcherShutdown()
		taskmanager.Shutdown()
	}()

	// Set GC percent if not configured
	if os.Getenv(envGOGC) == "" {
		// trigger GC when heap grows 15% above the previous live set
		debug.SetGCPercent(15)
	}
	runtime.SystemdNotify(true, "running")

	waitDone := make(chan struct{})
	go func() {
		wg.Wait()
		close(waitDone)
	}()

	go func() {
		<-waitDone
		close(errChan)
	}()

	// Wait for either:


   438→	// Wait for either:
   439→	// 1. An error from server startup
   440→	// 2. Completion of all goroutines (normal shutdown or crash)
   441→	select {
   442→	case err := <-errChan:
   443→		// errChan will be closed when waitDone is closed, which happens
   444→		// when all goroutines complete (either from normal shutdown or error)
   445→		if err != nil {
   446→			return err
   447→		}
   448→	case <-time.After(100 * time.Millisecond):
   449→		// Give the server 100ms to start and report any immediate startup errors
   450→		// After that, just wait for normal shutdown completion
   451→		select {
   452→		case err := <-errChan:
   453→			if err != nil {
   454→				return err
   455→			}
   456→		case <-waitDone:
   457→			// Normal shutdown completed

	// Wait for either:
	// 1. An error from server startup
	// 2. Completion of all goroutines (normal shutdown or crash)
	select {
	case err := <-errChan:
		// errChan will be closed when waitDone is closed, which happens
		// when all goroutines complete (either from normal shutdown or error)
		if err != nil {
			return err
		}
	case <-time.After(100 * time.Millisecond):
		// Give the server 100ms to start and report any immediate startup errors
		// After that, just wait for normal shutdown completion
		select {
		case err := <-errChan:
			if err != nil {
				return err
			}
		case <-waitDone:
			// Normal shutdown completed


Now let me look at the `metricstore.Shutdown()` and how the server's `Shutdown` interacts with the WaitGroup:


pkg/metricstore/metricstore.go:273:func Shutdown() {


   170→	ctx, shutdown := context.WithCancel(context.Background())
   171→
   172→	Retention(wg, ctx)
   173→	Checkpointing(wg, ctx)
   174→	CleanUp(wg, ctx)
   175→	WALStaging(wg, ctx)
   176→	MemoryUsageTracker(wg, ctx)
   177→
   178→	// Note: Signal handling has been removed from this function.
   179→	// The caller is responsible for handling shutdown signals and calling
   180→	// the shutdown() function when appropriate.
   181→	// Store the shutdown function for later use by Shutdown()
   182→	shutdownFuncMu.Lock()
   183→	shutdownFunc = shutdown
   184→	shutdownFuncMu.Unlock()
   185→
   186→	if Keys.Subscriptions != nil {
   187→		wg.Go(func() {
   188→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   189→				cclog.Fatal(err)
   190→			}
   191→		})
   192→	}
   193→}
   194→
   195→// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
   196→//
   197→// This function must be called before GetMemoryStore() and can only be called once due to
   198→// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
   199→//
   200→// Parameters:
   201→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
   202→//
   203→// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
   204→//
   205→// After this call, the global msInstance is ready for use via GetMemoryStore().
   206→func InitMetrics(metrics map[string]MetricConfig) {
   207→	singleton.Do(func() {
   208→		offset := 0
   209→		for key, cfg := range metrics {
   210→			if cfg.Frequency == 0 {
   211→				panic("[METRICSTORE]> invalid frequency")
   212→			}
   213→
   214→			metrics[key] = MetricConfig{
   215→				Frequency:   cfg.Frequency,
   216→				Aggregation: cfg.Aggregation,
   217→				offset:      offset,
   218→			}
   219→			offset += 1
   220→		}
   221→
   222→		msInstance = &MemoryStore{
   223→			root: Level{
   224→				metrics:  make([]*buffer, len(metrics)),
   225→				children: make(map[string]*Level),
   226→			},
   227→			Metrics: metrics,
   228→		}
   229→	})
   230→}
   231→
   232→// GetMemoryStore returns the singleton MemoryStore instance.
   233→//
   234→// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
   235→//
   236→// This function is safe for concurrent use after initialization.
   237→func GetMemoryStore() *MemoryStore {
   238→	if msInstance == nil {
   239→		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
   240→	}
   241→
   242→	return msInstance
   243→}
   244→
   245→func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
   246→	if metric, ok := ms.Metrics[metricName]; ok {
   247→		return metric.Frequency, nil
   248→	}
   249→	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
   250→}
   251→
   252→// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
   253→// This must be called during initialization to provide job state information
   254→// for selective buffer retention during Free operations.
   255→// If not set, the Free function will fall back to freeing all buffers.
   256→func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
   257→	ms.nodeProvider = provider
   258→}
   259→
   260→// Shutdown performs a graceful shutdown of the metric store.
   261→//
   262→// This function cancels all background goroutines started by Init() and writes
   263→// a final checkpoint to disk before returning. It should be called when the
   264→// application receives a termination signal.
   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→
   290→	ms := GetMemoryStore()
   291→
   292→	lastCheckpointMu.Lock()
   293→	from := lastCheckpoint
   294→	lastCheckpointMu.Unlock()
   295→
   296→	if Keys.Checkpoints.FileFormat == "wal" {
   297→		var hostDirs []string
   298→		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   299→		if err == nil {

	ctx, shutdown := context.WithCancel(context.Background())

	Retention(wg, ctx)
	Checkpointing(wg, ctx)
	CleanUp(wg, ctx)
	WALStaging(wg, ctx)
	MemoryUsageTracker(wg, ctx)

	// Note: Signal handling has been removed from this function.
	// The caller is responsible for handling shutdown signals and calling
	// the shutdown() function when appropriate.
	// Store the shutdown function for later use by Shutdown()
	shutdownFuncMu.Lock()
	shutdownFunc = shutdown
	shutdownFuncMu.Unlock()

	if Keys.Subscriptions != nil {
		wg.Go(func() {
			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
				cclog.Fatal(err)
			}
		})
	}
}

// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
//
// This function must be called before GetMemoryStore() and can only be called once due to
// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
//
// Parameters:
//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
//
// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
//
// After this call, the global msInstance is ready for use via GetMemoryStore().
func InitMetrics(metrics map[string]MetricConfig) {
	singleton.Do(func() {
		offset := 0
		for key, cfg := range metrics {
			if cfg.Frequency == 0 {
				panic("[METRICSTORE]> invalid frequency")
			}

			metrics[key] = MetricConfig{
				Frequency:   cfg.Frequency,
				Aggregation: cfg.Aggregation,
				offset:      offset,
			}
			offset += 1
		}

		msInstance = &MemoryStore{
			root: Level{
				metrics:  make([]*buffer, len(metrics)),
				children: make(map[string]*Level),
			},
			Metrics: metrics,
		}
	})
}

// GetMemoryStore returns the singleton MemoryStore instance.
//
// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
//
// This function is safe for concurrent use after initialization.
func GetMemoryStore() *MemoryStore {
	if msInstance == nil {
		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
	}

	return msInstance
}

func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
	if metric, ok := ms.Metrics[metricName]; ok {
		return metric.Frequency, nil
	}
	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
}

// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
// This must be called during initialization to provide job state information
// for selective buffer retention during Free operations.
// If not set, the Free function will fall back to freeing all buffers.
func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
	ms.nodeProvider = provider
}

// Shutdown performs a graceful shutdown of the metric store.
//
// This function cancels all background goroutines started by Init() and writes
// a final checkpoint to disk before returning. It should be called when the
// application receives a termination signal.
//
// The function will:
//  1. Cancel the context to stop all background workers
//  2. Close the WAL messages channel if using WAL format
//  3. Write a final checkpoint to preserve in-memory data
//  4. Log any errors encountered during shutdown
//
// Note: This function blocks until the final checkpoint is written.
func Shutdown() {
	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}

	if Keys.Checkpoints.FileFormat == "wal" {
		for _, ch := range walShardChs {
			close(ch)
		}
	}

	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		var hostDirs []string
		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
		if err == nil {


Let me check what the server shutdown looked like BEFORE the "Improve shutdown time" commit:


git show 93a9d732^:cmd/cc-backend/server.go | tail -n +400

Server shutdown code before the change