<local-command-caveat>Caveat: The messages below were generated by the user while running local commands. DO NOT respond to these messages or otherwise consider them in your response unless the user explicitly asks you to.</local-command-caveat>


<command-name>/clear</command-name>
            <command-message>clear</command-message>
            <command-args></command-args>


<local-command-stdout></local-command-stdout>


After a regular checkpoint the following log warning is writte nfor all nodes:  [WARNING]  walCheckpoint.go:343: [METRICSTORE]> WAL rotation skipped for var/metric-checkpoints/fritz/f0748 (channel full)
Mar 28 06:39:43 monitoring cc-backend[45645]: [WARNING]  walCheckpoint.go:343: [METRICSTORE]> WAL rotation skipped for var/metric-checkpoints/fritz/f2263 (channel full)
Mar 28 06:39:43 monitoring cc-backend[45645]: [WARNING]  walCheckpoint.go:343: [METRICSTORE]> WAL rotation skipped for var/metric-checkpoints/fritz/f0481 (channel full)
Mar 28 06:39:43 monitoring cc-backend[45645]: [WARNING]  walCheckpoint.go:343: [METRICSTORE]> WAL rotation skipped for var/metric-checkpoints/fritz/f0708 (channel full)
Mar 28 06:39:43 monitoring cc-backend[45645]: [WARNING]  walCheckpoint.go:343: [METRICSTORE]> WAL rotation skipped for var/metric-checkpoints/fritz/f0373 (channel full) . Investigate and issue and suggest fixes.


Error: File content (10285 tokens) exceeds maximum allowed tokens (10000). Use offset and limit parameters to read specific portions of the file, or search for specific content instead of reading the whole file.


Grep

WAL rotation skipped|channel full|RotateWAL|walRotation


pkg/metricstore/checkpoint.go-134-						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-135-						lastCheckpointMu.Lock()
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-136-						lastCheckpoint = now
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-137-						lastCheckpointMu.Unlock()
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-138-						// Rotate WAL files for successfully checkpointed hosts.
pkg/metricstore/checkpoint.go:139:						RotateWALFiles(hostDirs)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-140-					}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-141-				} else {
pkg/metricstore/checkpoint.go-142-					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-143-					if err != nil {
pkg/metricstore/checkpoint.go-144-						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
--
pkg/metricstore/lineprotocol.go-360-				Selector:   append([]string{}, st.selector...),
pkg/metricstore/lineprotocol.go-361-				Value:      metric.Value,
pkg/metricstore/lineprotocol.go-362-				Timestamp:  time,
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/lineprotocol.go-363-			}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/lineprotocol.go-364-			if !SendWALMessage(msg) {
pkg/metricstore/lineprotocol.go:365:				// WAL shard channel full — metric is written to memory store but not WAL.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/lineprotocol.go-366-				// Next binary snapshot will capture it.
pkg/metricstore/lineprotocol.go-367-				if dropped := walDropped.Add(1); dropped%10000 == 1 {
pkg/metricstore/lineprotocol.go:368:					cclog.Warnf("[METRICSTORE]> WAL shard channel full, dropped %d messages (data safe in memory, next snapshot will capture)", dropped)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/lineprotocol.go-369-				}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/lineprotocol.go-370-			}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/lineprotocol.go-371-		}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/lineprotocol.go-372-
pkg/metricstore/lineprotocol.go-373-		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-346-// flushed and closed before checkpoint writes begin.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-347-func WaitForWALStagingDrain() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-348-	walStagingWg.Wait()
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-349-}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-350-
pkg/metricstore/walCheckpoint.go:351:// RotateWALFiles sends rotation requests for the given host directories
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-352-// and blocks until all rotations complete. Each request is routed to the
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-353-// shard that owns the host directory.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-354-//
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-355-// If shutdown is in progress (WAL staging goroutines may have exited),
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-356-// rotation is skipped to avoid deadlocking on abandoned channels.
pkg/metricstore/walCheckpoint.go:357:func RotateWALFiles(hostDirs []string) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-358-	if walShardRotateChs == nil || walShuttingDown.Load() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-359-		return
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-360-	}
pkg/metricstore/walCheckpoint.go-361-	dones := make([]chan struct{}, 0, len(hostDirs))
pkg/metricstore/walCheckpoint.go-362-	for _, dir := range hostDirs {
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-365-		select {
pkg/metricstore/walCheckpoint.go-366-		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-367-			dones = append(dones, done)
pkg/metricstore/walCheckpoint.go-368-		default:
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-369-			// Channel full or goroutine not consuming — skip this host.
pkg/metricstore/walCheckpoint.go:370:			cclog.Warnf("[METRICSTORE]> WAL rotation skipped for %s (channel full)", dir)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-371-		}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-372-	}
pkg/metricstore/walCheckpoint.go-373-	for _, done := range dones {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-374-		select {
pkg/metricstore/walCheckpoint.go-375-		case <-done:
--
pkg/metricstore/walCheckpoint.go-387-	node := path.Base(hostDir)
pkg/metricstore/walCheckpoint.go-388-	cluster := path.Base(path.Dir(hostDir))
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-389-	return walShardIndex(cluster, node)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-390-}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-391-
pkg/metricstore/walCheckpoint.go:392:// RotateWALFiles sends rotation requests for the given host directories
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-393-// and blocks until all rotations complete.
pkg/metricstore/walCheckpoint.go:394:func RotateWALFilesAfterShutdown(hostDirs []string) {
pkg/metricstore/walCheckpoint.go-395-	for _, dir := range hostDirs {
pkg/metricstore/walCheckpoint.go-396-		walPath := path.Join(dir, "current.wal")
pkg/metricstore/walCheckpoint.go-397-		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
pkg/metricstore/walCheckpoint.go-398-			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-399-		}


1	// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
2	// All rights reserved. This file is part of cc-backend.
3	// Use of this source code is governed by a MIT-style
4	// license that can be found in the LICENSE file.
5
6	// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
7	//
8	// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
9	// per host, eliminating the extra memory overhead of the AvroStore and providing
10	// truly continuous (per-write) crash safety.
11	//
12	// # Architecture
13	//
14	//	Metric write (DecodeLine)
15	//	      │
16	//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
17	//	      │
18	//	      └─► WALMessages channel
19	//	               │
20	//	               ▼
21	//	        WALStaging goroutine
22	//	               │
23	//	               ▼
24	//	        checkpoints/cluster/host/current.wal  (append-only, binary)
25	//
26	//	Periodic checkpoint (Checkpointing goroutine):
27	//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
28	//	  2. Signal WALStaging to truncate current.wal per host
29	//
30	//	On restart (FromCheckpoint):
31	//	  1. Load most recent <timestamp>.bin snapshot
32	//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
33	//
34	// # WAL Record Format
35	//
36	//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
37	//
38	//	payload:
39	//	  [8B timestamp int64]
40	//	  [2B metric_name_len uint16][N metric name bytes]
41	//	  [1B selector_count uint8]
42	//	  per selector: [1B selector_len uint8][M selector bytes]
43	//	  [4B value float32 bits]
44	//
45	// # Binary Snapshot Format
46	//
47	//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
48	//	Level tree (recursive):
49	//	  [4B num_metrics uint32]
50	//	  per metric:
51	//	    [2B name_len uint16][N name bytes]
52	//	    [8B frequency int64][8B start int64]
53	//	    [4B num_values uint32][num_values × 4B float32]
54	//	  [4B num_children uint32]
55	//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
56	package metricstore
57
58	import (
59		"bufio"
60		"context"
61		"encoding/binary"
62		"fmt"
63		"hash/crc32"
64		"hash/fnv"
65		"io"
66		"math"
67		"os"
68		"path"
69		"strings"
70		"sync"
71		"sync/atomic"
72		"time"
73
74		cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
75		"github.com/ClusterCockpit/cc-lib/v2/schema"
76	)
77
78	// Magic numbers for binary formats.
79	const (
80		walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
81		walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
82		snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
83	)
84
85	// walShardChs holds per-shard channels for WAL messages.
86	// Initialized by WALStaging; nil when WAL is not active.
87	var walShardChs []chan *WALMessage
88
89	// walShardRotateChs holds per-shard channels for WAL rotation requests.
90	var walShardRotateChs []chan walRotateReq
91
92	// walNumShards stores the number of shards (set during WALStaging init).
93	var walNumShards int
94
95	// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
96	var walStagingWg sync.WaitGroup
97
98	// walShuttingDown is set before closing shard channels to prevent
99	// SendWALMessage from sending on a closed channel (which panics in Go).
100	var walShuttingDown atomic.Bool

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼
//	        WALStaging goroutine
//	               │
//	               ▼
//	        checkpoints/cluster/host/current.wal  (append-only, binary)
//
//	Periodic checkpoint (Checkpointing goroutine):
//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
//	  2. Signal WALStaging to truncate current.wal per host
//
//	On restart (FromCheckpoint):
//	  1. Load most recent <timestamp>.bin snapshot
//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
//
// # WAL Record Format
//
//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
//
//	payload:
//	  [8B timestamp int64]
//	  [2B metric_name_len uint16][N metric name bytes]
//	  [1B selector_count uint8]
//	  per selector: [1B selector_len uint8][M selector bytes]
//	  [4B value float32 bits]
//
// # Binary Snapshot Format
//
//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
//	Level tree (recursive):
//	  [4B num_metrics uint32]
//	  per metric:
//	    [2B name_len uint16][N name bytes]
//	    [8B frequency int64][8B start int64]
//	    [4B num_values uint32][num_values × 4B float32]
//	  [4B num_children uint32]
//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
package metricstore

import (
	"bufio"
	"context"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"hash/fnv"
	"io"
	"math"
	"os"
	"path"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Magic numbers for binary formats.
const (
	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
)

// walShardChs holds per-shard channels for WAL messages.
// Initialized by WALStaging; nil when WAL is not active.
var walShardChs []chan *WALMessage

// walShardRotateChs holds per-shard channels for WAL rotation requests.
var walShardRotateChs []chan walRotateReq

// walNumShards stores the number of shards (set during WALStaging init).
var walNumShards int

// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
var walStagingWg sync.WaitGroup

// walShuttingDown is set before closing shard channels to prevent
// SendWALMessage from sending on a closed channel (which panics in Go).
var walShuttingDown atomic.Bool


100	var walShuttingDown atomic.Bool
101
102	// WALMessage represents a single metric write to be appended to the WAL.
103	// Cluster and Node are NOT stored in the WAL record (inferred from file path).
104	type WALMessage struct {
105		MetricName string
106		Cluster    string
107		Node       string
108		Selector   []string
109		Value      schema.Float
110		Timestamp  int64
111	}
112
113	// walRotateReq requests WAL file rotation for a specific host directory.
114	// The done channel is closed by the WAL goroutine when rotation is complete.
115	type walRotateReq struct {
116		hostDir string
117		done    chan struct{}
118	}
119
120	// walFileState holds an open WAL file handle and buffered writer for one host directory.
121	type walFileState struct {
122		f     *os.File
123		w     *bufio.Writer
124		dirty bool
125		size  int64 // approximate bytes written (tracked from open + writes)
126	}
127
128	// walFlushInterval controls how often dirty WAL files are flushed to disk.
129	// Decoupling flushes from message processing lets the consumer run at memory
130	// speed, amortizing syscall overhead across many writes.
131	const walFlushInterval = 1 * time.Second
132
133	// walShardIndex computes which shard a message belongs to based on cluster+node.
134	// Uses FNV-1a hash for fast, well-distributed mapping.
135	func walShardIndex(cluster, node string) int {
136		h := fnv.New32a()
137		h.Write([]byte(cluster))
138		h.Write([]byte{0})
139		h.Write([]byte(node))
140		return int(h.Sum32()) % walNumShards
141	}
142
143	// SendWALMessage routes a WAL message to the appropriate shard channel.
144	// Returns false if the channel is full or shutdown is in progress.
145	func SendWALMessage(msg *WALMessage) bool {
146		if walShardChs == nil || walShuttingDown.Load() {
147			return false
148		}
149		shard := walShardIndex(msg.Cluster, msg.Node)
150		select {
151		case walShardChs[shard] <- msg:
152			return true
153		default:
154			return false
155		}
156	}
157
158	// WALStaging starts N sharded background goroutines that receive WALMessage items
159	// and append binary WAL records to per-host current.wal files.
160	// Each shard owns a subset of hosts (determined by hash of cluster+node),
161	// parallelizing serialization and I/O while keeping per-host writes serial.
162	func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
163		if Keys.Checkpoints.FileFormat == "json" {
164			return
165		}
166
167		walNumShards = max(Keys.NumWorkers, 1)
168		chBufSize := max(65536/walNumShards, 1024)
169
170		walShardChs = make([]chan *WALMessage, walNumShards)
171		walShardRotateChs = make([]chan walRotateReq, walNumShards)
172
173		for i := range walNumShards {
174			walShardChs[i] = make(chan *WALMessage, chBufSize)
175			walShardRotateChs[i] = make(chan walRotateReq, 64)
176		}
177
178		for i := range walNumShards {
179			msgCh := walShardChs[i]
180			rotateCh := walShardRotateChs[i]
181
182			walStagingWg.Add(1)
183			wg.Go(func() {
184				defer walStagingWg.Done()
185				hostFiles := make(map[string]*walFileState)
186
187				defer func() {
188					for _, ws := range hostFiles {
189						if ws.f != nil {
190							ws.w.Flush()
191							ws.f.Close()
192						}
193					}
194				}()
195
196				getOrOpenWAL := func(hostDir string) *walFileState {
197					ws, ok := hostFiles[hostDir]
198					if ok {
199						return ws

var walShuttingDown atomic.Bool

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f     *os.File
	w     *bufio.Writer
	dirty bool
	size  int64 // approximate bytes written (tracked from open + writes)
}

// walFlushInterval controls how often dirty WAL files are flushed to disk.
// Decoupling flushes from message processing lets the consumer run at memory
// speed, amortizing syscall overhead across many writes.
const walFlushInterval = 1 * time.Second

// walShardIndex computes which shard a message belongs to based on cluster+node.
// Uses FNV-1a hash for fast, well-distributed mapping.
func walShardIndex(cluster, node string) int {
	h := fnv.New32a()
	h.Write([]byte(cluster))
	h.Write([]byte{0})
	h.Write([]byte(node))
	return int(h.Sum32()) % walNumShards
}

// SendWALMessage routes a WAL message to the appropriate shard channel.
// Returns false if the channel is full or shutdown is in progress.
func SendWALMessage(msg *WALMessage) bool {
	if walShardChs == nil || walShuttingDown.Load() {
		return false
	}
	shard := walShardIndex(msg.Cluster, msg.Node)
	select {
	case walShardChs[shard] <- msg:
		return true
	default:
		return false
	}
}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.
// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return
	}

	walNumShards = max(Keys.NumWorkers, 1)
	chBufSize := max(65536/walNumShards, 1024)

	walShardChs = make([]chan *WALMessage, walNumShards)
	walShardRotateChs = make([]chan walRotateReq, walNumShards)

	for i := range walNumShards {
		walShardChs[i] = make(chan *WALMessage, chBufSize)
		walShardRotateChs[i] = make(chan walRotateReq, 64)
	}

	for i := range walNumShards {
		msgCh := walShardChs[i]
		rotateCh := walShardRotateChs[i]

		walStagingWg.Add(1)
		wg.Go(func() {
			defer walStagingWg.Done()
			hostFiles := make(map[string]*walFileState)

			defer func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
						ws.f.Close()
					}
				}
			}()

			getOrOpenWAL := func(hostDir string) *walFileState {
				ws, ok := hostFiles[hostDir]
				if ok {
					return ws


200					}
201
202					if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
203						cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
204						return nil
205					}
206
207					walPath := path.Join(hostDir, "current.wal")
208					f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
209					if err != nil {
210						cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
211						return nil
212					}
213
214					w := bufio.NewWriter(f)
215
216					// Write file header magic if file is new (empty).
217					info, err := f.Stat()
218					var fileSize int64
219					if err == nil {
220						fileSize = info.Size()
221					}
222					if err == nil && fileSize == 0 {
223						var hdr [4]byte
224						binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
225						if _, err := w.Write(hdr[:]); err != nil {
226							cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
227							f.Close()
228							return nil
229						}
230						fileSize = 4
231					}
232
233					ws = &walFileState{f: f, w: w, size: fileSize}
234					hostFiles[hostDir] = ws
235					return ws
236				}
237
238				processMsg := func(msg *WALMessage) {
239					hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
240					ws := getOrOpenWAL(hostDir)
241					if ws == nil {
242						return
243					}
244
245					// Enforce max WAL size: force-rotate before writing if limit is exceeded.
246					// The in-memory store still holds the data; only crash-recovery coverage is lost.
247					if maxSize := Keys.Checkpoints.MaxWALSize; maxSize > 0 && ws.size >= maxSize {
248						cclog.Warnf("[METRICSTORE]> WAL: force-rotating %s (size %d >= limit %d)",
249							hostDir, ws.size, maxSize)
250						ws.w.Flush()
251						ws.f.Close()
252						walPath := path.Join(hostDir, "current.wal")
253						if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
254							cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
255						}
256						delete(hostFiles, hostDir)
257						ws = getOrOpenWAL(hostDir)
258						if ws == nil {
259							return
260						}
261					}
262
263					n, err := writeWALRecordDirect(ws.w, msg)
264					if err != nil {
265						cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
266					}
267					ws.size += int64(n)
268					ws.dirty = true
269				}
270
271				processRotate := func(req walRotateReq) {
272					ws, ok := hostFiles[req.hostDir]
273					if ok && ws.f != nil {
274						ws.w.Flush()
275						ws.f.Close()
276						walPath := path.Join(req.hostDir, "current.wal")
277						if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
278							cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
279						}
280						delete(hostFiles, req.hostDir)
281					}
282					close(req.done)
283				}
284
285				flushDirty := func() {
286					for _, ws := range hostFiles {
287						if ws.dirty {
288							ws.w.Flush()
289							ws.dirty = false
290						}
291					}
292				}
293
294				ticker := time.NewTicker(walFlushInterval)
295				defer ticker.Stop()
296
297				// drainBatch processes up to 4096 pending messages without blocking.
298				// Returns false if the channel was closed.
299				drainBatch := func() bool {

				}

				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
					return nil
				}

				walPath := path.Join(hostDir, "current.wal")
				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
					return nil
				}

				w := bufio.NewWriter(f)

				// Write file header magic if file is new (empty).
				info, err := f.Stat()
				var fileSize int64
				if err == nil {
					fileSize = info.Size()
				}
				if err == nil && fileSize == 0 {
					var hdr [4]byte
					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
					if _, err := w.Write(hdr[:]); err != nil {
						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
						f.Close()
						return nil
					}
					fileSize = 4
				}

				ws = &walFileState{f: f, w: w, size: fileSize}
				hostFiles[hostDir] = ws
				return ws
			}

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}

				// Enforce max WAL size: force-rotate before writing if limit is exceeded.
				// The in-memory store still holds the data; only crash-recovery coverage is lost.
				if maxSize := Keys.Checkpoints.MaxWALSize; maxSize > 0 && ws.size >= maxSize {
					cclog.Warnf("[METRICSTORE]> WAL: force-rotating %s (size %d >= limit %d)",
						hostDir, ws.size, maxSize)
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, hostDir)
					ws = getOrOpenWAL(hostDir)
					if ws == nil {
						return
					}
				}

				n, err := writeWALRecordDirect(ws.w, msg)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
				ws.size += int64(n)
				ws.dirty = true
			}

			processRotate := func(req walRotateReq) {
				ws, ok := hostFiles[req.hostDir]
				if ok && ws.f != nil {
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(req.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, req.hostDir)
				}
				close(req.done)
			}

			flushDirty := func() {
				for _, ws := range hostFiles {
					if ws.dirty {
						ws.w.Flush()
						ws.dirty = false
					}
				}
			}

			ticker := time.NewTicker(walFlushInterval)
			defer ticker.Stop()

			// drainBatch processes up to 4096 pending messages without blocking.
			// Returns false if the channel was closed.
			drainBatch := func() bool {


300					for range 4096 {
301						select {
302						case msg, ok := <-msgCh:
303							if !ok {
304								flushDirty()
305								return false
306							}
307							processMsg(msg)
308						case req := <-rotateCh:
309							processRotate(req)
310						default:
311							return true
312						}
313					}
314					return true
315				}
316
317				for {
318					select {
319					case <-ctx.Done():
320						// On shutdown, skip draining buffered messages — a full binary
321						// checkpoint will be written from in-memory state, making
322						// buffered WAL records redundant.
323						flushDirty()
324						return
325					case msg, ok := <-msgCh:
326						if !ok {
327							return
328						}
329						processMsg(msg)
330						if !drainBatch() {
331							return
332						}
333						// No flush here — timer handles periodic flushing.
334					case <-ticker.C:
335						flushDirty()
336					case req := <-rotateCh:
337						processRotate(req)
338					}
339				}
340			})
341		}
342	}
343
344	// WaitForWALStagingDrain blocks until all WALStaging goroutines have exited.
345	// Must be called after closing walShardChs to ensure all file handles are
346	// flushed and closed before checkpoint writes begin.
347	func WaitForWALStagingDrain() {
348		walStagingWg.Wait()
349	}
350
351	// RotateWALFiles sends rotation requests for the given host directories
352	// and blocks until all rotations complete. Each request is routed to the
353	// shard that owns the host directory.
354	//
355	// If shutdown is in progress (WAL staging goroutines may have exited),
356	// rotation is skipped to avoid deadlocking on abandoned channels.
357	func RotateWALFiles(hostDirs []string) {
358		if walShardRotateChs == nil || walShuttingDown.Load() {
359			return
360		}
361		dones := make([]chan struct{}, 0, len(hostDirs))
362		for _, dir := range hostDirs {
363			done := make(chan struct{})
364			shard := walShardIndexFromDir(dir)
365			select {
366			case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
367				dones = append(dones, done)
368			default:
369				// Channel full or goroutine not consuming — skip this host.
370				cclog.Warnf("[METRICSTORE]> WAL rotation skipped for %s (channel full)", dir)
371			}
372		}
373		for _, done := range dones {
374			select {
375			case <-done:
376			case <-time.After(30 * time.Second):
377				cclog.Warn("[METRICSTORE]> WAL rotation timed out, continuing")
378				return
379			}
380		}
381	}
382
383	// walShardIndexFromDir extracts cluster and node from a host directory path
384	// and returns the shard index. The path format is: rootDir/cluster/node
385	func walShardIndexFromDir(hostDir string) int {
386		// hostDir = .../cluster/node — extract last two path components
387		node := path.Base(hostDir)
388		cluster := path.Base(path.Dir(hostDir))
389		return walShardIndex(cluster, node)
390	}
391
392	// RotateWALFiles sends rotation requests for the given host directories
393	// and blocks until all rotations complete.
394	func RotateWALFilesAfterShutdown(hostDirs []string) {
395		for _, dir := range hostDirs {
396			walPath := path.Join(dir, "current.wal")
397			if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
398				cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
399			}
400		}
401	}
402
403	// writeWALRecordDirect encodes a WAL record into a contiguous buffer first,
404	// then writes it to the bufio.Writer in a single call. This prevents partial
405	// records in the write buffer if a write error occurs mid-record (e.g. disk full).
406	// Returns the number of bytes written and any error.
407	func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) (int, error) {
408		// Compute payload and total record size.
409		payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4

				for range 4096 {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							flushDirty()
							return false
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						return true
					}
				}
				return true
			}

			for {
				select {
				case <-ctx.Done():
					// On shutdown, skip draining buffered messages — a full binary
					// checkpoint will be written from in-memory state, making
					// buffered WAL records redundant.
					flushDirty()
					return
				case msg, ok := <-msgCh:
					if !ok {
						return
					}
					processMsg(msg)
					if !drainBatch() {
						return
					}
					// No flush here — timer handles periodic flushing.
				case <-ticker.C:
					flushDirty()
				case req := <-rotateCh:
					processRotate(req)
				}
			}
		})
	}
}

// WaitForWALStagingDrain blocks until all WALStaging goroutines have exited.
// Must be called after closing walShardChs to ensure all file handles are
// flushed and closed before checkpoint writes begin.
func WaitForWALStagingDrain() {
	walStagingWg.Wait()
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete. Each request is routed to the
// shard that owns the host directory.
//
// If shutdown is in progress (WAL staging goroutines may have exited),
// rotation is skipped to avoid deadlocking on abandoned channels.
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		default:
			// Channel full or goroutine not consuming — skip this host.
			cclog.Warnf("[METRICSTORE]> WAL rotation skipped for %s (channel full)", dir)
		}
	}
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation timed out, continuing")
			return
		}
	}
}

// walShardIndexFromDir extracts cluster and node from a host directory path
// and returns the shard index. The path format is: rootDir/cluster/node
func walShardIndexFromDir(hostDir string) int {
	// hostDir = .../cluster/node — extract last two path components
	node := path.Base(hostDir)
	cluster := path.Base(path.Dir(hostDir))
	return walShardIndex(cluster, node)
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete.
func RotateWALFilesAfterShutdown(hostDirs []string) {
	for _, dir := range hostDirs {
		walPath := path.Join(dir, "current.wal")
		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
		}
	}
}

// writeWALRecordDirect encodes a WAL record into a contiguous buffer first,
// then writes it to the bufio.Writer in a single call. This prevents partial
// records in the write buffer if a write error occurs mid-record (e.g. disk full).
// Returns the number of bytes written and any error.
func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) (int, error) {
	// Compute payload and total record size.
	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4


Now I have a clear picture. Let me check how many hosts are typically involved and the checkpoint flow.


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-136-						lastCheckpoint = now
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-137-						lastCheckpointMu.Unlock()
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-138-						// Rotate WAL files for successfully checkpointed hosts.
pkg/metricstore/checkpoint.go:139:						RotateWALFiles(hostDirs)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-140-					}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-141-				} else {
pkg/metricstore/checkpoint.go-142-					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-354-//
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-355-// If shutdown is in progress (WAL staging goroutines may have exited),
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-356-// rotation is skipped to avoid deadlocking on abandoned channels.
pkg/metricstore/walCheckpoint.go:357:func RotateWALFiles(hostDirs []string) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-358-	if walShardRotateChs == nil || walShuttingDown.Load() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-359-		return
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-360-	}


126-				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
127-
128-				if Keys.Checkpoints.FileFormat == "wal" {
129:					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
130-					if err != nil {
131-						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
132-					}
--
136-						lastCheckpoint = now
137-						lastCheckpointMu.Unlock()
138-						// Rotate WAL files for successfully checkpointed hosts.
139:						RotateWALFiles(hostDirs)
140-					}
141-				} else {
142-					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-650-}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-651-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-652-// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-653-// Returns the number of files written, the list of host directories that were
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-654-// successfully checkpointed (for WAL rotation), and any errors.
pkg/metricstore/walCheckpoint.go:655:func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-656-	// Collect all cluster/host pairs.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-657-	m.root.lock.RLock()
pkg/metricstore/walCheckpoint.go-658-	totalHosts := 0
pkg/metricstore/walCheckpoint.go-659-	for _, l1 := range m.root.children {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-660-		l1.lock.RLock()


Now let me check how many hosts we're dealing with (the `NumWorkers` config that determines shard count).


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-134-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-135-
pkg/metricstore/metricstore.go:136:	// Set NumWorkers from config or use default
pkg/metricstore/metricstore.go:137:	if Keys.NumWorkers <= 0 {
pkg/metricstore/metricstore.go:138:		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-139-	}
pkg/metricstore/metricstore.go:140:	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-141-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-142-	// Pass the config.MetricStoreKeys
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-186-	if Keys.Subscriptions != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-187-		wg.Go(func() {
pkg/metricstore/metricstore.go:188:			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-189-				cclog.Fatal(err)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-190-			}
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-107-	var wg sync.WaitGroup
pkg/metricstore/archive.go-108-	n, errs := int32(0), int32(0)
pkg/metricstore/archive.go:109:	work := make(chan workItem, Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-110-
pkg/metricstore/archive.go:111:	wg.Add(Keys.NumWorkers)
pkg/metricstore/archive.go:112:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-113-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-114-			defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-206-		}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-207-
pkg/metricstore/archive.go:208:		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
pkg/metricstore/archive.go-209-		results := make(chan hostResult, 2)
pkg/metricstore/archive.go-210-		work := make(chan struct {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-211-			dir, host string
pkg/metricstore/archive.go:212:		}, Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-213-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-214-		var wg sync.WaitGroup
pkg/metricstore/archive.go-215-		errs := int32(0)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-216-
pkg/metricstore/archive.go:217:		wg.Add(Keys.NumWorkers)
pkg/metricstore/archive.go:218:		for w := 0; w < Keys.NumWorkers; w++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-219-			go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-220-				defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-183-//
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-184-// Metrics at root and cluster levels are skipped. One file per host is created.
pkg/metricstore/checkpoint.go:185:// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-186-// at a time, allowing concurrent writes/reads to other hosts.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-187-//
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-221-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-222-	var wg sync.WaitGroup
pkg/metricstore/checkpoint.go:223:	wg.Add(Keys.NumWorkers)
pkg/metricstore/checkpoint.go:224:	work := make(chan workItem, Keys.NumWorkers*2)
pkg/metricstore/checkpoint.go:225:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-226-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-227-			defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-396-func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-397-	var wg sync.WaitGroup
pkg/metricstore/checkpoint.go:398:	work := make(chan [2]string, Keys.NumWorkers*4)
pkg/metricstore/checkpoint.go-399-	n, errs := int32(0), int32(0)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-400-
pkg/metricstore/checkpoint.go:401:	wg.Add(Keys.NumWorkers)
pkg/metricstore/checkpoint.go:402:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-403-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-404-			defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-165-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-166-
pkg/metricstore/walCheckpoint.go:167:	walNumShards = max(Keys.NumWorkers, 1)
pkg/metricstore/walCheckpoint.go-168-	chBufSize := max(65536/walNumShards, 1024)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-169-
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-685-
pkg/metricstore/walCheckpoint.go-686-	totalWork := len(levels)
pkg/metricstore/walCheckpoint.go:687:	cclog.Infof("[METRICSTORE]> Starting binary checkpoint for %d hosts with %d workers", totalWork, Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-688-
pkg/metricstore/walCheckpoint.go-689-	n, errs, completed := int32(0), int32(0), int32(0)
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-692-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-693-	var wg sync.WaitGroup
pkg/metricstore/walCheckpoint.go:694:	wg.Add(Keys.NumWorkers)
pkg/metricstore/walCheckpoint.go:695:	work := make(chan workItem, Keys.NumWorkers*2)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-696-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-697-	// Progress logging goroutine.
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-711-	}()
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-712-
pkg/metricstore/walCheckpoint.go:713:	for range Keys.NumWorkers {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-714-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-715-			defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-11-//
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-12-//	MetricStoreConfig (Keys)
pkg/metricstore/config.go:13://	├─ NumWorkers: Parallel checkpoint/archive workers
pkg/metricstore/config.go-14-/	├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval)
pkg/metricstore/config.go-15-/	├─ MemoryCap: Memory limit in bytes (triggers forceFree)
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-115-//
pkg/metricstore/config.go-116-/ Fields:
pkg/metricstore/config.go:117://   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
pkg/metricstore/config.go-118-/   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
pkg/metricstore/config.go-119-/   - MemoryCap:         Max memory in GB for buffer data (0 = unlimited); triggers forceFree when exceeded
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-125-	// Number of concurrent workers for checkpoint and archive operations.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-126-	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
pkg/metricstore/config.go:127:	NumWorkers         int            `json:"num-workers"`
pkg/metricstore/config.go-128-	RetentionInMemory  string         `json:"retention-in-memory"`
pkg/metricstore/config.go-129-	CheckpointInterval string         `json:"checkpoint-interval,omitempty"`


Assume thousands of nodes


I now have a clear understanding of the issue. Let me write the plan.


Write

# Fix: WAL rotation "channel full" warnings for all nodes

## Context

After every checkpoint, **all** nodes log `WAL rotation skipped for ... (channel full)`. This means WAL files are never rotated (deleted after a successful binary snapshot), causing unbounded WAL file growth — effectively defeating the purpose of checkpointing.

## Root Cause

In `pkg/metricstore/walCheckpoint.go`:

1. **Rotation channels are small** (line 175): `walShardRotateChs[i] = make(chan walRotateReq, 64)` — buffer of 64 per shard.
2. **Shards are few** (line 167): `walNumShards = max(Keys.NumWorkers, 1)` — typically 5-10 shards.
3. **Hundreds of hosts per shard**: A large cluster like "fritz" has many hundreds of nodes. With ~10 shards, each shard handles ~100+ hosts.
4. **Non-blocking send** (lines 365-371): `RotateWALFiles` uses `select { case ch <- req: ... default: skip }`. It sends all rotation requests in a tight loop without waiting.
5. **Slow consumer**: The WAL staging goroutine processes one rotation at a time (flush + close + delete = I/O), interleaved with incoming metric messages.

Result: The first 64 rotation requests per shard succeed (fill the buffer), then every subsequent host is skipped with the warning. Since the staging goroutine can't drain 64 I/O-bound rotations fast enough during the tight loop, nearly all nodes are skipped.

## Fix

**Change the non-blocking send to a blocking send with a timeout** in `RotateWALFiles` (line 365-371).

The checkpoint goroutine is a background task — it can afford to wait. This ensures all hosts get their WAL rotated.

### File to modify

`pkg/metricstore/walCheckpoint.go` — `RotateWALFiles` function (lines 357-381)

### Change

Replace the non-blocking `select` with a blocking send + timeout:

```go
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		case <-time.After(30 * time.Second):
			cclog.Warnf("[METRICSTORE]> WAL rotation send timed out for %s", dir)
		}
	}
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation timed out, continuing")
			return
		}
	}
}
```

Key change: `default:` (non-blocking skip) becomes `case <-time.After(30 * time.Second):` (blocking with timeout). This allows the staging goroutine time to drain the channel.

**Note**: Using `time.After` in a loop over potentially thousands of hosts creates a timer per iteration. Since this runs once per checkpoint interval (not a hot path), this is acceptable. If preferred, a single shared deadline can be used instead:

```go
deadline := time.After(2 * time.Minute) // single deadline for all sends
for _, dir := range hostDirs {
    done := make(chan struct{})
    shard := walShardIndexFromDir(dir)
    select {
    case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
        dones = append(dones, done)
    case <-deadline:
        cclog.Warnf("[METRICSTORE]> WAL rotation send timed out after deadline, %d hosts remaining", len(hostDirs)-len(dones))
        goto waitDones
    }
}
```

I'd recommend the single-deadline approach for cleanliness.

## Verification

1. Build: `go build ./pkg/metricstore/...`
2. Run tests: `go test ./pkg/metricstore/...`
3. Deploy and observe: after checkpoint, no more "channel full" warnings; WAL files should be deleted for all hosts after each successful checkpoint.


File created successfully at: /Users/jan/.claude/plans/warm-enchanting-clarke.md

# Fix: WAL rotation "channel full" warnings for all nodes

## Context

After every checkpoint, **all** nodes log `WAL rotation skipped for ... (channel full)`. This means WAL files are never rotated (deleted after a successful binary snapshot), causing unbounded WAL file growth — effectively defeating the purpose of checkpointing.

## Root Cause

In `pkg/metricstore/walCheckpoint.go`:

1. **Rotation channels are small** (line 175): `walShardRotateChs[i] = make(chan walRotateReq, 64)` — buffer of 64 per shard.
2. **Shards are few** (line 167): `walNumShards = max(Keys.NumWorkers, 1)` — typically 5-10 shards.
3. **Hundreds of hosts per shard**: A large cluster like "fritz" has many hundreds of nodes. With ~10 shards, each shard handles ~100+ hosts.
4. **Non-blocking send** (lines 365-371): `RotateWALFiles` uses `select { case ch <- req: ... default: skip }`. It sends all rotation requests in a tight loop without waiting.
5. **Slow consumer**: The WAL staging goroutine processes one rotation at a time (flush + close + delete = I/O), interleaved with incoming metric messages.

Result: The first 64 rotation requests per shard succeed (fill the buffer), then every subsequent host is skipped with the warning. Since the staging goroutine can't drain 64 I/O-bound rotations fast enough during the tight loop, nearly all nodes are skipped.

## Fix

**Change the non-blocking send to a blocking send with a timeout** in `RotateWALFiles` (line 365-371).

The checkpoint goroutine is a background task — it can afford to wait. This ensures all hosts get their WAL rotated.

### File to modify

`pkg/metricstore/walCheckpoint.go` — `RotateWALFiles` function (lines 357-381)

### Change

Replace the non-blocking `select` with a blocking send + timeout:

```go
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		case <-time.After(30 * time.Second):
			cclog.Warnf("[METRICSTORE]> WAL rotation send timed out for %s", dir)
		}
	}
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation timed out, continuing")
			return
		}
	}
}
```

Key change: `default:` (non-blocking skip) becomes `case <-time.After(30 * time.Second):` (blocking with timeout). This allows the staging goroutine time to drain the channel.

**Note**: Using `time.After` in a loop over potentially thousands of hosts creates a timer per iteration. Since this runs once per checkpoint interval (not a hot path), this is acceptable. If preferred, a single shared deadline can be used instead:

```go
deadline := time.After(2 * time.Minute) // single deadline for all sends
for _, dir := range hostDirs {
    done := make(chan struct{})
    shard := walShardIndexFromDir(dir)
    select {
    case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
        dones = append(dones, done)
    case <-deadline:
        cclog.Warnf("[METRICSTORE]> WAL rotation send timed out after deadline, %d hosts remaining", len(hostDirs)-len(dones))
        goto waitDones
    }
}
```

I'd recommend the single-deadline approach for cleanliness.

## Verification

1. Build: `go build ./pkg/metricstore/...`
2. Run tests: `go test ./pkg/metricstore/...`
3. Deploy and observe: after checkpoint, no more "channel full" warnings; WAL files should be deleted for all hosts after each successful checkpoint.


Good — with thousands of nodes, the issue is even more acute. Let me refine the plan slightly.


Edit

I'd recommend the single-deadline approach for cleanliness.

## Verification

I'd recommend the single-deadline approach for cleanliness.

### Additional consideration: rotation starvation under load

The WAL staging goroutine's `drainBatch()` processes up to 4096 messages before yielding back to the main `select` (which can then pick a rotation request). With thousands of nodes continuously sending metrics, the rotation channel may drain slowly because the goroutine alternates between large message batches and individual rotations.

With a blocking send + 2-minute deadline this should still work (each rotation is fast I/O — flush/close/delete), but if it proves tight, we could also process pending rotation requests inside `drainBatch()`. For now, the blocking-send fix alone should resolve the issue.

## Verification


The file /Users/jan/.claude/plans/warm-enchanting-clarke.md has been updated successfully.

I'd recommend the single-deadline approach for cleanliness.

### Additional consideration: rotation starvation under load

The WAL staging goroutine's `drainBatch()` processes up to 4096 messages before yielding back to the main `select` (which can then pick a rotation request). With thousands of nodes continuously sending metrics, the rotation channel may drain slowly because the goroutine alternates between large message batches and individual rotations.

With a blocking send + 2-minute deadline this should still work (each rotation is fast I/O — flush/close/delete), but if it proves tight, we could also process pending rotation requests inside `drainBatch()`. For now, the blocking-send fix alone should resolve the issue.

## Verification

# Fix: WAL rotation "channel full" warnings for all nodes

## Context

After every checkpoint, **all** nodes log `WAL rotation skipped for ... (channel full)`. This means WAL files are never rotated (deleted after a successful binary snapshot), causing unbounded WAL file growth — effectively defeating the purpose of checkpointing.

## Root Cause

In `pkg/metricstore/walCheckpoint.go`:

1. **Rotation channels are small** (line 175): `walShardRotateChs[i] = make(chan walRotateReq, 64)` — buffer of 64 per shard.
2. **Shards are few** (line 167): `walNumShards = max(Keys.NumWorkers, 1)` — typically 5-10 shards.
3. **Hundreds of hosts per shard**: A large cluster like "fritz" has many hundreds of nodes. With ~10 shards, each shard handles ~100+ hosts.
4. **Non-blocking send** (lines 365-371): `RotateWALFiles` uses `select { case ch <- req: ... default: skip }`. It sends all rotation requests in a tight loop without waiting.
5. **Slow consumer**: The WAL staging goroutine processes one rotation at a time (flush + close + delete = I/O), interleaved with incoming metric messages.

Result: The first 64 rotation requests per shard succeed (fill the buffer), then every subsequent host is skipped with the warning. Since the staging goroutine can't drain 64 I/O-bound rotations fast enough during the tight loop, nearly all nodes are skipped.

## Fix

**Change the non-blocking send to a blocking send with a timeout** in `RotateWALFiles` (line 365-371).

The checkpoint goroutine is a background task — it can afford to wait. This ensures all hosts get their WAL rotated.

### File to modify

`pkg/metricstore/walCheckpoint.go` — `RotateWALFiles` function (lines 357-381)

### Change

Replace the non-blocking `select` with a blocking send + timeout:

```go
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		case <-time.After(30 * time.Second):
			cclog.Warnf("[METRICSTORE]> WAL rotation send timed out for %s", dir)
		}
	}
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation timed out, continuing")
			return
		}
	}
}
```

Key change: `default:` (non-blocking skip) becomes `case <-time.After(30 * time.Second):` (blocking with timeout). This allows the staging goroutine time to drain the channel.

**Note**: Using `time.After` in a loop over potentially thousands of hosts creates a timer per iteration. Since this runs once per checkpoint interval (not a hot path), this is acceptable. If preferred, a single shared deadline can be used instead:

```go
deadline := time.After(2 * time.Minute) // single deadline for all sends
for _, dir := range hostDirs {
    done := make(chan struct{})
    shard := walShardIndexFromDir(dir)
    select {
    case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
        dones = append(dones, done)
    case <-deadline:
        cclog.Warnf("[METRICSTORE]> WAL rotation send timed out after deadline, %d hosts remaining", len(hostDirs)-len(dones))
        goto waitDones
    }
}
```

I'd recommend the single-deadline approach for cleanliness.

## Verification

1. Build: `go build ./pkg/metricstore/...`
2. Run tests: `go test ./pkg/metricstore/...`
3. Deploy and observe: after checkpoint, no more "channel full" warnings; WAL files should be deleted for all hosts after each successful checkpoint.

+### Additional consideration: rotation starvation under load

+The WAL staging goroutine's `drainBatch()` processes up to 4096 messages before yielding back to the main `select` (which can then pick a rotation request). With thousands of nodes continuously sending metrics, the rotation channel may drain slowly because the goroutine alternates between large message batches and individual rotations.

+With a blocking send + 2-minute deadline this should still work (each rotation is fast I/O — flush/close/delete), but if it proves tight, we could also process pending rotation requests inside `drainBatch()`. For now, the blocking-send fix alone should resolve the issue.


ToolSearch

Question: select:ExitPlanMode


Question: select:ExitPlanMode


The root cause is clear: `RotateWALFiles` uses a **non-blocking send** (`default:` case) on rotation channels buffered at only 64. With thousands of nodes and ~10 shards, the tight loop fills each shard's buffer instantly, and every remaining host is skipped. WAL files are never deleted.

The fix is simple: change the non-blocking send to a **blocking send with a timeout**, so the checkpoint goroutine waits for the staging goroutine to drain the channel instead of immediately giving up.


ExitPlanMode

Prompt: build Go packages

Prompt: run Go tests

# Fix: WAL rotation "channel full" warnings for all nodes

## Context

After every checkpoint, **all** nodes log `WAL rotation skipped for ... (channel full)`. This means WAL files are never rotated (deleted after a successful binary snapshot), causing unbounded WAL file growth — effectively defeating the purpose of checkpointing.

## Root Cause

In `pkg/metricstore/walCheckpoint.go`:

1. **Rotation channels are small** (line 175): `walShardRotateChs[i] = make(chan walRotateReq, 64)` — buffer of 64 per shard.
2. **Shards are few** (line 167): `walNumShards = max(Keys.NumWorkers, 1)` — typically 5-10 shards.
3. **Hundreds of hosts per shard**: A large cluster like "fritz" has many hundreds of nodes. With ~10 shards, each shard handles ~100+ hosts.
4. **Non-blocking send** (lines 365-371): `RotateWALFiles` uses `select { case ch <- req: ... default: skip }`. It sends all rotation requests in a tight loop without waiting.
5. **Slow consumer**: The WAL staging goroutine processes one rotation at a time (flush + close + delete = I/O), interleaved with incoming metric messages.

Result: The first 64 rotation requests per shard succeed (fill the buffer), then every subsequent host is skipped with the warning. Since the staging goroutine can't drain 64 I/O-bound rotations fast enough during the tight loop, nearly all nodes are skipped.

## Fix

**Change the non-blocking send to a blocking send with a timeout** in `RotateWALFiles` (line 365-371).

The checkpoint goroutine is a background task — it can afford to wait. This ensures all hosts get their WAL rotated.

### File to modify

`pkg/metricstore/walCheckpoint.go` — `RotateWALFiles` function (lines 357-381)

### Change

Replace the non-blocking `select` with a blocking send + timeout:

```go
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		case <-time.After(30 * time.Second):
			cclog.Warnf("[METRICSTORE]> WAL rotation send timed out for %s", dir)
		}
	}
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation timed out, continuing")
			return
		}
	}
}
```

Key change: `default:` (non-blocking skip) becomes `case <-time.After(30 * time.Second):` (blocking with timeout). This allows the staging goroutine time to drain the channel.

**Note**: Using `time.After` in a loop over potentially thousands of hosts creates a timer per iteration. Since this runs once per checkpoint interval (not a hot path), this is acceptable. If preferred, a single shared deadline can be used instead:

```go
deadline := time.After(2 * time.Minute) // single deadline for all sends
for _, dir := range hostDirs {
    done := make(chan struct{})
    shard := walShardIndexFromDir(dir)
    select {
    case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
        dones = append(dones, done)
    case <-deadline:
        cclog.Warnf("[METRICSTORE]> WAL rotation send timed out after deadline, %d hosts remaining", len(hostDirs)-len(dones))
        goto waitDones
    }
}
```

I'd recommend the single-deadline approach for cleanliness.

### Additional consideration: rotation starvation under load

The WAL staging goroutine's `drainBatch()` processes up to 4096 messages before yielding back to the main `select` (which can then pick a rotation request). With thousands of nodes continuously sending metrics, the rotation channel may drain slowly because the goroutine alternates between large message batches and individual rotations.

With a blocking send + 2-minute deadline this should still work (each rotation is fast I/O — flush/close/delete), but if it proves tight, we could also process pending rotation requests inside `drainBatch()`. For now, the blocking-send fix alone should resolve the issue.

## Verification

1. Build: `go build ./pkg/metricstore/...`
2. Run tests: `go test ./pkg/metricstore/...`
3. Deploy and observe: after checkpoint, no more "channel full" warnings; WAL files should be deleted for all hosts after each successful checkpoint.


User has approved your plan. You can now start coding. Start with updating your todo list if applicable

Your plan has been saved to: /Users/jan/.claude/plans/warm-enchanting-clarke.md
You can refer back to it if needed during implementation.

## Approved Plan:
# Fix: WAL rotation "channel full" warnings for all nodes

## Context

After every checkpoint, **all** nodes log `WAL rotation skipped for ... (channel full)`. This means WAL files are never rotated (deleted after a successful binary snapshot), causing unbounded WAL file growth — effectively defeating the purpose of checkpointing.

## Root Cause

In `pkg/metricstore/walCheckpoint.go`:

1. **Rotation channels are small** (line 175): `walShardRotateChs[i] = make(chan walRotateReq, 64)` — buffer of 64 per shard.
2. **Shards are few** (line 167): `walNumShards = max(Keys.NumWorkers, 1)` — typically 5-10 shards.
3. **Hundreds of hosts per shard**: A large cluster like "fritz" has many hundreds of nodes. With ~10 shards, each shard handles ~100+ hosts.
4. **Non-blocking send** (lines 365-371): `RotateWALFiles` uses `select { case ch <- req: ... default: skip }`. It sends all rotation requests in a tight loop without waiting.
5. **Slow consumer**: The WAL staging goroutine processes one rotation at a time (flush + close + delete = I/O), interleaved with incoming metric messages.

Result: The first 64 rotation requests per shard succeed (fill the buffer), then every subsequent host is skipped with the warning. Since the staging goroutine can't drain 64 I/O-bound rotations fast enough during the tight loop, nearly all nodes are skipped.

## Fix

**Change the non-blocking send to a blocking send with a timeout** in `RotateWALFiles` (line 365-371).

The checkpoint goroutine is a background task — it can afford to wait. This ensures all hosts get their WAL rotated.

### File to modify

`pkg/metricstore/walCheckpoint.go` — `RotateWALFiles` function (lines 357-381)

### Change

Replace the non-blocking `select` with a blocking send + timeout:

```go
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		case <-time.After(30 * time.Second):
			cclog.Warnf("[METRICSTORE]> WAL rotation send timed out for %s", dir)
		}
	}
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation timed out, continuing")
			return
		}
	}
}
```

Key change: `default:` (non-blocking skip) becomes `case <-time.After(30 * time.Second):` (blocking with timeout). This allows the staging goroutine time to drain the channel.

**Note**: Using `time.After` in a loop over potentially thousands of hosts creates a timer per iteration. Since this runs once per checkpoint interval (not a hot path), this is acceptable. If preferred, a single shared deadline can be used instead:

```go
deadline := time.After(2 * time.Minute) // single deadline for all sends
for _, dir := range hostDirs {
    done := make(chan struct{})
    shard := walShardIndexFromDir(dir)
    select {
    case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
        dones = append(dones, done)
    case <-deadline:
        cclog.Warnf("[METRICSTORE]> WAL rotation send timed out after deadline, %d hosts remaining", len(hostDirs)-len(dones))
        goto waitDones
    }
}
```

I'd recommend the single-deadline approach for cleanliness.

### Additional consideration: rotation starvation under load

The WAL staging goroutine's `drainBatch()` processes up to 4096 messages before yielding back to the main `select` (which can then pick a rotation request). With thousands of nodes continuously sending metrics, the rotation channel may drain slowly because the goroutine alternates between large message batches and individual rotations.

With a blocking send + 2-minute deadline this should still work (each rotation is fast I/O — flush/close/delete), but if it proves tight, we could also process pending rotation requests inside `drainBatch()`. For now, the blocking-send fix alone should resolve the issue.

## Verification

1. Build: `go build ./pkg/metricstore/...`
2. Run tests: `go test ./pkg/metricstore/...`
3. Deploy and observe: after checkpoint, no more "channel full" warnings; WAL files should be deleted for all hosts after each successful checkpoint.


Now let me implement the fix.


Edit

func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		default:
			// Channel full or goroutine not consuming — skip this host.
			cclog.Warnf("[METRICSTORE]> WAL rotation skipped for %s (channel full)", dir)
		}
	}
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation timed out, continuing")
			return
		}
	}
}

func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil || walShuttingDown.Load() {
		return
	}
	deadline := time.After(2 * time.Minute)
	dones := make([]chan struct{}, 0, len(hostDirs))
	for _, dir := range hostDirs {
		done := make(chan struct{})
		shard := walShardIndexFromDir(dir)
		select {
		case walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: done}:
			dones = append(dones, done)
		case <-deadline:
			cclog.Warnf("[METRICSTORE]> WAL rotation send timed out, %d of %d hosts remaining",
				len(hostDirs)-len(dones), len(hostDirs))
			goto waitDones
		}
	}
waitDones:
	for _, done := range dones {
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			cclog.Warn("[METRICSTORE]> WAL rotation completion timed out, continuing")
			return
		}
	}
}