   145→// Each shard owns a subset of hosts (determined by hash of cluster+node),
   146→// parallelizing serialization and I/O while keeping per-host writes serial.
   147→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   148→	if Keys.Checkpoints.FileFormat == "json" {
   149→		return
   150→	}
   151→
   152→	walNumShards = max(Keys.NumWorkers, 1)
   153→	chBufSize := max(65536/walNumShards, 1024)
   154→
   155→	walShardChs = make([]chan *WALMessage, walNumShards)
   156→	walShardRotateChs = make([]chan walRotateReq, walNumShards)
   157→
   158→	for i := range walNumShards {
   159→		walShardChs[i] = make(chan *WALMessage, chBufSize)
   160→		walShardRotateChs[i] = make(chan walRotateReq, 64)
   161→	}
   162→
   163→	for i := range walNumShards {
   164→		msgCh := walShardChs[i]
   165→		rotateCh := walShardRotateChs[i]
   166→
   167→		wg.Go(func() {
   168→			hostFiles := make(map[string]*walFileState)
   169→
   170→			defer func() {
   171→				for _, ws := range hostFiles {
   172→					if ws.f != nil {
   173→						ws.w.Flush()
   174→						ws.f.Close()
   175→					}
   176→				}
   177→			}()
   178→
   179→			getOrOpenWAL := func(hostDir string) *walFileState {
   180→				ws, ok := hostFiles[hostDir]
   181→				if ok {
   182→					return ws
   183→				}
   184→
   185→				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   186→					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   187→					return nil
   188→				}
   189→
   190→				walPath := path.Join(hostDir, "current.wal")
   191→				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   192→				if err != nil {
   193→					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   194→					return nil

// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return
	}

	walNumShards = max(Keys.NumWorkers, 1)
	chBufSize := max(65536/walNumShards, 1024)

	walShardChs = make([]chan *WALMessage, walNumShards)
	walShardRotateChs = make([]chan walRotateReq, walNumShards)

	for i := range walNumShards {
		walShardChs[i] = make(chan *WALMessage, chBufSize)
		walShardRotateChs[i] = make(chan walRotateReq, 64)
	}

	for i := range walNumShards {
		msgCh := walShardChs[i]
		rotateCh := walShardRotateChs[i]

		wg.Go(func() {
			hostFiles := make(map[string]*walFileState)

			defer func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
						ws.f.Close()
					}
				}
			}()

			getOrOpenWAL := func(hostDir string) *walFileState {
				ws, ok := hostFiles[hostDir]
				if ok {
					return ws
				}

				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
					return nil
				}

				walPath := path.Join(hostDir, "current.wal")
				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
					return nil


   350→			return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
   351→		}
   352→
   353→		time := t.Unix()
   354→
   355→		if Keys.Checkpoints.FileFormat == "wal" {
   356→			msg := &WALMessage{
   357→				MetricName: string(st.metricBuf),
   358→				Cluster:    cluster,
   359→				Node:       host,
   360→				Selector:   append([]string{}, st.selector...),
   361→				Value:      metric.Value,
   362→				Timestamp:  time,
   363→			}
   364→			if !SendWALMessage(msg) {
   365→				// WAL shard channel full — metric is written to memory store but not WAL.
   366→				// Next binary snapshot will capture it.
   367→				if dropped := walDropped.Add(1); dropped%10000 == 1 {
   368→					cclog.Warnf("[METRICSTORE]> WAL shard channel full, dropped %d messages (data safe in memory, next snapshot will capture)", dropped)
   369→				}
   370→			}
   371→		}
   372→
   373→		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
   374→			cclog.Warnf("write error for host %s metric %s at ts %d: %s", host, string(st.metricBuf), time, err.Error())
   375→			continue
   376→		}
   377→	}
   378→	return nil
   379→}

			return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
		}

		time := t.Unix()

		if Keys.Checkpoints.FileFormat == "wal" {
			msg := &WALMessage{
				MetricName: string(st.metricBuf),
				Cluster:    cluster,
				Node:       host,
				Selector:   append([]string{}, st.selector...),
				Value:      metric.Value,
				Timestamp:  time,
			}
			if !SendWALMessage(msg) {
				// WAL shard channel full — metric is written to memory store but not WAL.
				// Next binary snapshot will capture it.
				if dropped := walDropped.Add(1); dropped%10000 == 1 {
					cclog.Warnf("[METRICSTORE]> WAL shard channel full, dropped %d messages (data safe in memory, next snapshot will capture)", dropped)
				}
			}
		}

		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
			cclog.Warnf("write error for host %s metric %s at ts %d: %s", host, string(st.metricBuf), time, err.Error())
			continue
		}
	}
	return nil
}


   230→					ws.w.Flush()
   231→					ws.f.Close()
   232→					walPath := path.Join(req.hostDir, "current.wal")
   233→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   234→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   235→					}
   236→					delete(hostFiles, req.hostDir)
   237→				}
   238→				close(req.done)
   239→			}
   240→
   241→			flushAll := func() {
   242→				for _, ws := range hostFiles {
   243→					if ws.f != nil {
   244→						ws.w.Flush()
   245→					}
   246→				}
   247→			}
   248→
   249→			drain := func() {
   250→				for {
   251→					select {
   252→					case msg, ok := <-msgCh:
   253→						if !ok {
   254→							return
   255→						}
   256→						processMsg(msg)
   257→					case req := <-rotateCh:
   258→						processRotate(req)
   259→					default:
   260→						flushAll()
   261→						return
   262→					}
   263→				}
   264→			}
   265→
   266→			for {
   267→				select {
   268→				case <-ctx.Done():
   269→					drain()
   270→					return
   271→				case msg, ok := <-msgCh:
   272→					if !ok {
   273→						return
   274→					}
   275→					processMsg(msg)
   276→
   277→					// Drain up to 256 more messages without blocking to batch writes.
   278→					for range 256 {
   279→						select {

					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(req.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, req.hostDir)
				}
				close(req.done)
			}

			flushAll := func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
					}
				}
			}

			drain := func() {
				for {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							return
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						flushAll()
						return
					}
				}
			}

			for {
				select {
				case <-ctx.Done():
					drain()
					return
				case msg, ok := <-msgCh:
					if !ok {
						return
					}
					processMsg(msg)

					// Drain up to 256 more messages without blocking to batch writes.
					for range 256 {
						select {


   278→					for range 256 {
   279→						select {
   280→						case msg, ok := <-msgCh:
   281→							if !ok {
   282→								return
   283→							}
   284→							processMsg(msg)
   285→						case req := <-rotateCh:
   286→							processRotate(req)
   287→						default:
   288→							goto flushed
   289→						}
   290→					}
   291→				flushed:
   292→					flushAll()
   293→				case req := <-rotateCh:
   294→					processRotate(req)
   295→				}
   296→			}
   297→		})
   298→	}
   299→}
   300→
   301→// RotateWALFiles sends rotation requests for the given host directories
   302→// and blocks until all rotations complete. Each request is routed to the
   303→// shard that owns the host directory.
   304→func RotateWALFiles(hostDirs []string) {
   305→	if walShardRotateChs == nil {
   306→		return
   307→	}

					for range 256 {
						select {
						case msg, ok := <-msgCh:
							if !ok {
								return
							}
							processMsg(msg)
						case req := <-rotateCh:
							processRotate(req)
						default:
							goto flushed
						}
					}
				flushed:
					flushAll()
				case req := <-rotateCh:
					processRotate(req)
				}
			}
		})
	}
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete. Each request is routed to the
// shard that owns the host directory.
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil {
		return
	}


pkg/metricstore/walCheckpoint.go:344:func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-345-	// Compute payload size.
pkg/metricstore/walCheckpoint.go-346-	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
pkg/metricstore/walCheckpoint.go-347-	for _, s := range msg.Selector {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-348-		payloadSize += 1 + len(s)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-349-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-350-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-351-	// Write magic + payload length (8 bytes header).
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-352-	var hdr [8]byte
pkg/metricstore/walCheckpoint.go-353-	binary.LittleEndian.PutUint32(hdr[0:4], walRecordMagic)
pkg/metricstore/walCheckpoint.go-354-	binary.LittleEndian.PutUint32(hdr[4:8], uint32(payloadSize))
pkg/metricstore/walCheckpoint.go-355-	if _, err := w.Write(hdr[:]); err != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-356-		return err
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-357-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-358-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-359-	// We need to compute CRC over the payload as we write it.
pkg/metricstore/walCheckpoint.go-360-	crc := crc32.NewIEEE()
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-361-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-362-	// Timestamp (8 bytes).
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-363-	var scratch [8]byte
pkg/metricstore/walCheckpoint.go-364-	binary.LittleEndian.PutUint64(scratch[:8], uint64(msg.Timestamp))


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-134-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-135-
pkg/metricstore/metricstore.go:136:	// Set NumWorkers from config or use default
pkg/metricstore/metricstore.go:137:	if Keys.NumWorkers <= 0 {
pkg/metricstore/metricstore.go:138:		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-139-	}
pkg/metricstore/metricstore.go:140:	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-141-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-142-	// Pass the config.MetricStoreKeys
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-186-	if Keys.Subscriptions != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-187-		wg.Go(func() {
pkg/metricstore/metricstore.go:188:			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-189-				cclog.Fatal(err)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-190-			}
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-182-//
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-183-// Metrics at root and cluster levels are skipped. One file per host is created.
pkg/metricstore/checkpoint.go:184:// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-185-// at a time, allowing concurrent writes/reads to other hosts.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-186-//
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-220-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-221-	var wg sync.WaitGroup
pkg/metricstore/checkpoint.go:222:	wg.Add(Keys.NumWorkers)
pkg/metricstore/checkpoint.go:223:	work := make(chan workItem, Keys.NumWorkers*2)
pkg/metricstore/checkpoint.go:224:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-225-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-226-			defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-395-func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-396-	var wg sync.WaitGroup
pkg/metricstore/checkpoint.go:397:	work := make(chan [2]string, Keys.NumWorkers*4)
pkg/metricstore/checkpoint.go-398-	n, errs := int32(0), int32(0)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-399-
pkg/metricstore/checkpoint.go:400:	wg.Add(Keys.NumWorkers)
pkg/metricstore/checkpoint.go:401:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-402-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-403-			defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-107-	var wg sync.WaitGroup
pkg/metricstore/archive.go-108-	n, errs := int32(0), int32(0)
pkg/metricstore/archive.go:109:	work := make(chan workItem, Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-110-
pkg/metricstore/archive.go:111:	wg.Add(Keys.NumWorkers)
pkg/metricstore/archive.go:112:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-113-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-114-			defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-206-		}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-207-
pkg/metricstore/archive.go:208:		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
pkg/metricstore/archive.go-209-		results := make(chan hostResult, 2)
pkg/metricstore/archive.go-210-		work := make(chan struct {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-211-			dir, host string
pkg/metricstore/archive.go:212:		}, Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-213-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-214-		var wg sync.WaitGroup
pkg/metricstore/archive.go-215-		errs := int32(0)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-216-
pkg/metricstore/archive.go:217:		wg.Add(Keys.NumWorkers)
pkg/metricstore/archive.go:218:		for w := 0; w < Keys.NumWorkers; w++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-219-			go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-220-				defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-150-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-151-
pkg/metricstore/walCheckpoint.go:152:	walNumShards = max(Keys.NumWorkers, 1)
pkg/metricstore/walCheckpoint.go-153-	chBufSize := max(65536/walNumShards, 1024)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-154-
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-703-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-704-	var wg sync.WaitGroup
pkg/metricstore/walCheckpoint.go:705:	wg.Add(Keys.NumWorkers)
pkg/metricstore/walCheckpoint.go:706:	work := make(chan workItem, Keys.NumWorkers*2)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-707-
pkg/metricstore/walCheckpoint.go:708:	for range Keys.NumWorkers {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-709-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-710-			defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-11-//
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-12-//	MetricStoreConfig (Keys)
pkg/metricstore/config.go:13://	├─ NumWorkers: Parallel checkpoint/archive workers
pkg/metricstore/config.go-14-/	├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval)
pkg/metricstore/config.go-15-/	├─ MemoryCap: Memory limit in bytes (triggers forceFree)
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-112-//
pkg/metricstore/config.go-113-/ Fields:
pkg/metricstore/config.go:114://   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
pkg/metricstore/config.go-115-/   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
pkg/metricstore/config.go-116-/   - MemoryCap:         Max memory in GB for buffer data (0 = unlimited); triggers forceFree when exceeded
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-122-	// Number of concurrent workers for checkpoint and archive operations.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-123-	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
pkg/metricstore/config.go:124:	NumWorkers         int            `json:"num-workers"`
pkg/metricstore/config.go-125-	RetentionInMemory  string         `json:"retention-in-memory"`
pkg/metricstore/config.go-126-	CheckpointInterval string         `json:"checkpoint-interval,omitempty"`


Now I have a clear picture. Let me check the `flushAll` bottleneck — it flushes every open file after each batch.


pkg/metricstore/metricstore.go:138:		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
pkg/metricstore/config.go:53:	DefaultMaxWorkers                 = 10


195-				}
196-
197:				w := bufio.NewWriter(f)
198-
199-				// Write file header magic if file is new (empty).
--
768-	}
769-
770:	bw := bufio.NewWriter(f)
771-	if err := writeBinarySnapshotFile(bw, cf); err != nil {
772-		f.Close()


Write

# Fix WAL shard channel overflow (dropped messages)

## Context

The WAL staging goroutines can't keep up with the metric ingestion rate, causing 7.2M+ messages to be dropped from WAL channels. Data is safe in memory and will be captured by the next binary snapshot, but WAL crash-recovery coverage is degraded between snapshots.

**Root cause**: `flushAll()` at `walCheckpoint.go:241-247` iterates over ALL open host WAL files and calls `Flush()` on every one — even files that received no new writes. On a large cluster with thousands of nodes, this means thousands of syscalls after every batch of ≤257 messages. The consumer goroutines spend most of their time in unnecessary flushes, so the channel backs up.

## Fix

**Track dirty files and only flush those.**

### Changes in `pkg/metricstore/walCheckpoint.go`

1. **Add a `dirty` flag to `walFileState`** (or use a separate `map`/`set` of dirty host dirs).

2. **In `processMsg`**: mark the host's WAL file as dirty after writing.

3. **Replace `flushAll()`** with a function that only flushes dirty files and clears their dirty flag.

This reduces flush work from O(total_hosts_in_shard) to O(hosts_written_in_batch) per batch cycle.

### Optional secondary improvement

Increase the hardcoded channel buffer from `65536` to a larger value (e.g., `262144`) to absorb bursts better. Or make it configurable via `Keys.Checkpoints`.

## Files to modify

- `pkg/metricstore/walCheckpoint.go` — `walFileState` struct, `processMsg`, `flushAll`, and optionally channel buffer size

## Verification

1. `go build ./...` — compiles cleanly
2. `go test ./pkg/metricstore/...` — existing tests pass
3. Deploy and monitor: the warning should disappear or appear far less frequently


File created successfully at: /Users/jan/.claude/plans/floating-scribbling-raven.md

# Fix WAL shard channel overflow (dropped messages)

## Context

The WAL staging goroutines can't keep up with the metric ingestion rate, causing 7.2M+ messages to be dropped from WAL channels. Data is safe in memory and will be captured by the next binary snapshot, but WAL crash-recovery coverage is degraded between snapshots.

**Root cause**: `flushAll()` at `walCheckpoint.go:241-247` iterates over ALL open host WAL files and calls `Flush()` on every one — even files that received no new writes. On a large cluster with thousands of nodes, this means thousands of syscalls after every batch of ≤257 messages. The consumer goroutines spend most of their time in unnecessary flushes, so the channel backs up.

## Fix

**Track dirty files and only flush those.**

### Changes in `pkg/metricstore/walCheckpoint.go`

1. **Add a `dirty` flag to `walFileState`** (or use a separate `map`/`set` of dirty host dirs).

2. **In `processMsg`**: mark the host's WAL file as dirty after writing.

3. **Replace `flushAll()`** with a function that only flushes dirty files and clears their dirty flag.

This reduces flush work from O(total_hosts_in_shard) to O(hosts_written_in_batch) per batch cycle.

### Optional secondary improvement

Increase the hardcoded channel buffer from `65536` to a larger value (e.g., `262144`) to absorb bursts better. Or make it configurable via `Keys.Checkpoints`.

## Files to modify

- `pkg/metricstore/walCheckpoint.go` — `walFileState` struct, `processMsg`, `flushAll`, and optionally channel buffer size

## Verification

1. `go build ./...` — compiles cleanly
2. `go test ./pkg/metricstore/...` — existing tests pass
3. Deploy and monitor: the warning should disappear or appear far less frequently


Here's the analysis:

**Is it a problem?** Partially. No data is lost — metrics are always written to memory first, and the next binary snapshot captures everything. However, the WAL's purpose is crash recovery between snapshots. With 7.2M dropped messages, the WAL provides essentially no crash protection during high-ingestion periods.

**Root cause**: `flushAll()` at `walCheckpoint.go:241-247` flushes **every** open host WAL file after each batch of ≤257 messages — even files that received no new writes in that batch. On a large cluster with thousands of nodes per shard, this means thousands of `bufio.Writer.Flush()` syscalls per batch cycle. The consumer goroutines spend most of their time doing unnecessary flushes, can't keep up, and the channel fills.

**Fix**: Track which files are dirty and only flush those. This reduces flush work from O(all_hosts_in_shard) to O(hosts_written_in_batch) per cycle — a dramatic improvement on large clusters.


ToolSearch

Question: select:ExitPlanMode


Question: select:ExitPlanMode


ExitPlanMode

Prompt: run go build and go test commands

# Fix WAL shard channel overflow (dropped messages)

## Context

The WAL staging goroutines can't keep up with the metric ingestion rate, causing 7.2M+ messages to be dropped from WAL channels. Data is safe in memory and will be captured by the next binary snapshot, but WAL crash-recovery coverage is degraded between snapshots.

**Root cause**: `flushAll()` at `walCheckpoint.go:241-247` iterates over ALL open host WAL files and calls `Flush()` on every one — even files that received no new writes. On a large cluster with thousands of nodes, this means thousands of syscalls after every batch of ≤257 messages. The consumer goroutines spend most of their time in unnecessary flushes, so the channel backs up.

## Fix

**Track dirty files and only flush those.**

### Changes in `pkg/metricstore/walCheckpoint.go`

1. **Add a `dirty` flag to `walFileState`** (or use a separate `map`/`set` of dirty host dirs).

2. **In `processMsg`**: mark the host's WAL file as dirty after writing.

3. **Replace `flushAll()`** with a function that only flushes dirty files and clears their dirty flag.

This reduces flush work from O(total_hosts_in_shard) to O(hosts_written_in_batch) per batch cycle.

### Optional secondary improvement

Increase the hardcoded channel buffer from `65536` to a larger value (e.g., `262144`) to absorb bursts better. Or make it configurable via `Keys.Checkpoints`.

## Files to modify

- `pkg/metricstore/walCheckpoint.go` — `walFileState` struct, `processMsg`, `flushAll`, and optionally channel buffer size

## Verification

1. `go build ./...` — compiles cleanly
2. `go test ./pkg/metricstore/...` — existing tests pass
3. Deploy and monitor: the warning should disappear or appear far less frequently


Error: The user doesn't want to proceed with this tool use. The tool use was rejected (eg. if it was a file edit, the new_string was NOT written to the file). To tell you how to proceed, the user said:
There is a constant stream of data for multiple clusters at the same rate. Does this change the plan?


Good point. With a constant stream (not bursty), increasing channel buffer sizes is useless — it just delays the overflow. The consumer throughput must match the sustained producer rate. Let me reconsider the bottleneck.


   195→				}
   196→
   197→				w := bufio.NewWriter(f)
   198→
   199→				// Write file header magic if file is new (empty).
   200→				info, err := f.Stat()
   201→				if err == nil && info.Size() == 0 {
   202→					var hdr [4]byte
   203→					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   204→					if _, err := w.Write(hdr[:]); err != nil {
   205→						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   206→						f.Close()
   207→						return nil
   208→					}
   209→				}
   210→
   211→				ws = &walFileState{f: f, w: w}
   212→				hostFiles[hostDir] = ws
   213→				return ws
   214→			}
   215→
   216→			processMsg := func(msg *WALMessage) {
   217→				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   218→				ws := getOrOpenWAL(hostDir)
   219→				if ws == nil {
   220→					return
   221→				}
   222→				if err := writeWALRecordDirect(ws.w, msg); err != nil {
   223→					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   224→				}
   225→			}
   226→
   227→			processRotate := func(req walRotateReq) {
   228→				ws, ok := hostFiles[req.hostDir]
   229→				if ok && ws.f != nil {
   230→					ws.w.Flush()
   231→					ws.f.Close()
   232→					walPath := path.Join(req.hostDir, "current.wal")
   233→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   234→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   235→					}
   236→					delete(hostFiles, req.hostDir)
   237→				}
   238→				close(req.done)
   239→			}
   240→
   241→			flushAll := func() {
   242→				for _, ws := range hostFiles {
   243→					if ws.f != nil {
   244→						ws.w.Flush()
   245→					}
   246→				}
   247→			}
   248→
   249→			drain := func() {
   250→				for {
   251→					select {
   252→					case msg, ok := <-msgCh:
   253→						if !ok {
   254→							return

				}

				w := bufio.NewWriter(f)

				// Write file header magic if file is new (empty).
				info, err := f.Stat()
				if err == nil && info.Size() == 0 {
					var hdr [4]byte
					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
					if _, err := w.Write(hdr[:]); err != nil {
						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
						f.Close()
						return nil
					}
				}

				ws = &walFileState{f: f, w: w}
				hostFiles[hostDir] = ws
				return ws
			}

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}
				if err := writeWALRecordDirect(ws.w, msg); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
			}

			processRotate := func(req walRotateReq) {
				ws, ok := hostFiles[req.hostDir]
				if ok && ws.f != nil {
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(req.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, req.hostDir)
				}
				close(req.done)
			}

			flushAll := func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
					}
				}
			}

			drain := func() {
				for {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							return


Now I understand the full picture. With constant-rate ingestion, the issue is different from burst handling. Let me update the plan.


Write

# Fix WAL shard channel overflow (dropped messages)

## Context

The WAL staging goroutines can't keep up with constant-rate metric ingestion across multiple clusters, causing 7.2M+ messages to be dropped from WAL channels. Data is safe in memory (captured by next binary snapshot), but WAL crash-recovery coverage between snapshots is degraded.

## Root cause

The main loop in `WALStaging` (`walCheckpoint.go:266-296`) processes a batch of up to 257 messages, then calls `flushAll()` which flushes **every** open `bufio.Writer` — one write syscall per dirty host file.

With constant-rate ingestion across many hosts, most hosts are dirty in every batch. So each cycle is: 257 fast in-memory writes → hundreds of write syscalls to flush. Flush time dominates, the consumer can't sustain the ingestion rate, and the channel fills permanently (buffer size is irrelevant for sustained rate).

## Fix: Timer-based flushing

**Decouple flushing from message processing.** Instead of flushing after every batch, flush dirty files on a periodic timer (e.g., every 1 second).

### Changes in `pkg/metricstore/walCheckpoint.go`

1. **Add a `dirty` flag** to `walFileState` struct. Set it in `processMsg` after writing.

2. **Replace `flushAll()` with `flushDirty()`** that only flushes files where `dirty == true` and clears the flag.

3. **Restructure the main loop** to use a `time.Ticker` for flush scheduling:

```go
ticker := time.NewTicker(walFlushInterval) // e.g., 1s
defer ticker.Stop()

for {
    select {
    case <-ctx.Done():
        drain()
        return
    case msg, ok := <-msgCh:
        if !ok { return }
        processMsg(msg)
        // Drain without blocking — larger batch (4096)
        for range 4096 {
            select {
            case msg, ok := <-msgCh:
                if !ok { flushDirty(); return }
                processMsg(msg)
            case req := <-rotateCh:
                processRotate(req)
            default:
                goto drained
            }
        }
    drained:
        // No flush here — timer handles it
    case <-ticker.C:
        flushDirty()
    case req := <-rotateCh:
        processRotate(req)
    }
}
```

**Why this works**: Messages are processed at memory speed (`bufio.Writer` absorbs writes into its 4KB buffer). Flushes happen once per second instead of after every 257 messages, amortizing the syscall cost. The consumer throughput increases dramatically.

**Flush interval**: 1 second is a good default — short enough for crash safety, long enough to amortize I/O. Could be made configurable later if needed.

4. **Increase batch drain** from 256 to 4096 — with no flush at the end, the batch just controls how long before we re-check ctx/rotate channels.

## Files to modify

- `pkg/metricstore/walCheckpoint.go` — `walFileState` struct, main loop in `WALStaging`, `flushAll` → `flushDirty`

## Verification

1. `go build ./...`
2. `go test ./pkg/metricstore/...`
3. Deploy and monitor: the dropped-message warning should stop appearing


The file /Users/jan/.claude/plans/floating-scribbling-raven.md has been updated successfully.

# Fix WAL shard channel overflow (dropped messages)

## Context

The WAL staging goroutines can't keep up with constant-rate metric ingestion across multiple clusters, causing 7.2M+ messages to be dropped from WAL channels. Data is safe in memory (captured by next binary snapshot), but WAL crash-recovery coverage between snapshots is degraded.

## Root cause

The main loop in `WALStaging` (`walCheckpoint.go:266-296`) processes a batch of up to 257 messages, then calls `flushAll()` which flushes **every** open `bufio.Writer` — one write syscall per dirty host file.

With constant-rate ingestion across many hosts, most hosts are dirty in every batch. So each cycle is: 257 fast in-memory writes → hundreds of write syscalls to flush. Flush time dominates, the consumer can't sustain the ingestion rate, and the channel fills permanently (buffer size is irrelevant for sustained rate).

## Fix: Timer-based flushing

**Decouple flushing from message processing.** Instead of flushing after every batch, flush dirty files on a periodic timer (e.g., every 1 second).

### Changes in `pkg/metricstore/walCheckpoint.go`

1. **Add a `dirty` flag** to `walFileState` struct. Set it in `processMsg` after writing.

2. **Replace `flushAll()` with `flushDirty()`** that only flushes files where `dirty == true` and clears the flag.

3. **Restructure the main loop** to use a `time.Ticker` for flush scheduling:

```go
ticker := time.NewTicker(walFlushInterval) // e.g., 1s
defer ticker.Stop()

for {
    select {
    case <-ctx.Done():
        drain()
        return
    case msg, ok := <-msgCh:
        if !ok { return }
        processMsg(msg)
        // Drain without blocking — larger batch (4096)
        for range 4096 {
            select {
            case msg, ok := <-msgCh:
                if !ok { flushDirty(); return }
                processMsg(msg)
            case req := <-rotateCh:
                processRotate(req)
            default:
                goto drained
            }
        }
    drained:
        // No flush here — timer handles it
    case <-ticker.C:
        flushDirty()
    case req := <-rotateCh:
        processRotate(req)
    }
}
```

**Why this works**: Messages are processed at memory speed (`bufio.Writer` absorbs writes into its 4KB buffer). Flushes happen once per second instead of after every 257 messages, amortizing the syscall cost. The consumer throughput increases dramatically.

**Flush interval**: 1 second is a good default — short enough for crash safety, long enough to amortize I/O. Could be made configurable later if needed.

4. **Increase batch drain** from 256 to 4096 — with no flush at the end, the batch just controls how long before we re-check ctx/rotate channels.

## Files to modify

- `pkg/metricstore/walCheckpoint.go` — `walFileState` struct, main loop in `WALStaging`, `flushAll` → `flushDirty`

## Verification

1. `go build ./...`
2. `go test ./pkg/metricstore/...`
3. Deploy and monitor: the dropped-message warning should stop appearing

-The WAL staging goroutines can't keep up with the metric ingestion rate, causing 7.2M+ messages to be dropped from WAL channels. Data is safe in memory and will be captured by the next binary snapshot, but WAL crash-recovery coverage is degraded between snapshots.

+The WAL staging goroutines can't keep up with constant-rate metric ingestion across multiple clusters, causing 7.2M+ messages to be dropped from WAL channels. Data is safe in memory (captured by next binary snapshot), but WAL crash-recovery coverage between snapshots is degraded.

-**Root cause**: `flushAll()` at `walCheckpoint.go:241-247` iterates over ALL open host WAL files and calls `Flush()` on every one — even files that received no new writes. On a large cluster with thousands of nodes, this means thousands of syscalls after every batch of ≤257 messages. The consumer goroutines spend most of their time in unnecessary flushes, so the channel backs up.

+The main loop in `WALStaging` (`walCheckpoint.go:266-296`) processes a batch of up to 257 messages, then calls `flushAll()` which flushes **every** open `bufio.Writer` — one write syscall per dirty host file.

-**Track dirty files and only flush those.**

+With constant-rate ingestion across many hosts, most hosts are dirty in every batch. So each cycle is: 257 fast in-memory writes → hundreds of write syscalls to flush. Flush time dominates, the consumer can't sustain the ingestion rate, and the channel fills permanently (buffer size is irrelevant for sustained rate).

+**Decouple flushing from message processing.** Instead of flushing after every batch, flush dirty files on a periodic timer (e.g., every 1 second).

-1. **Add a `dirty` flag to `walFileState`** (or use a separate `map`/`set` of dirty host dirs).

+1. **Add a `dirty` flag** to `walFileState` struct. Set it in `processMsg` after writing.

-2. **In `processMsg`**: mark the host's WAL file as dirty after writing.

+2. **Replace `flushAll()` with `flushDirty()`** that only flushes files where `dirty == true` and clears the flag.

-3. **Replace `flushAll()`** with a function that only flushes dirty files and clears their dirty flag.

+3. **Restructure the main loop** to use a `time.Ticker` for flush scheduling:

-This reduces flush work from O(total_hosts_in_shard) to O(hosts_written_in_batch) per batch cycle.

+ticker := time.NewTicker(walFlushInterval) // e.g., 1s

+        // Drain without blocking — larger batch (4096)

+                if !ok { flushDirty(); return }

+        // No flush here — timer handles it

-Increase the hardcoded channel buffer from `65536` to a larger value (e.g., `262144`) to absorb bursts better. Or make it configurable via `Keys.Checkpoints`.

+**Why this works**: Messages are processed at memory speed (`bufio.Writer` absorbs writes into its 4KB buffer). Flushes happen once per second instead of after every 257 messages, amortizing the syscall cost. The consumer throughput increases dramatically.

+**Flush interval**: 1 second is a good default — short enough for crash safety, long enough to amortize I/O. Could be made configurable later if needed.

+4. **Increase batch drain** from 256 to 4096 — with no flush at the end, the batch just controls how long before we re-check ctx/rotate channels.

-- `pkg/metricstore/walCheckpoint.go` — `walFileState` struct, `processMsg`, `flushAll`, and optionally channel buffer size

+- `pkg/metricstore/walCheckpoint.go` — `walFileState` struct, main loop in `WALStaging`, `flushAll` → `flushDirty`

-2. `go test ./pkg/metricstore/...` — existing tests pass

-3. Deploy and monitor: the warning should disappear or appear far less frequently

+3. Deploy and monitor: the dropped-message warning should stop appearing

# Fix WAL shard channel overflow (dropped messages)

## Context

The WAL staging goroutines can't keep up with the metric ingestion rate, causing 7.2M+ messages to be dropped from WAL channels. Data is safe in memory and will be captured by the next binary snapshot, but WAL crash-recovery coverage is degraded between snapshots.

**Root cause**: `flushAll()` at `walCheckpoint.go:241-247` iterates over ALL open host WAL files and calls `Flush()` on every one — even files that received no new writes. On a large cluster with thousands of nodes, this means thousands of syscalls after every batch of ≤257 messages. The consumer goroutines spend most of their time in unnecessary flushes, so the channel backs up.

## Fix

**Track dirty files and only flush those.**

### Changes in `pkg/metricstore/walCheckpoint.go`

1. **Add a `dirty` flag to `walFileState`** (or use a separate `map`/`set` of dirty host dirs).

2. **In `processMsg`**: mark the host's WAL file as dirty after writing.

3. **Replace `flushAll()`** with a function that only flushes dirty files and clears their dirty flag.

This reduces flush work from O(total_hosts_in_shard) to O(hosts_written_in_batch) per batch cycle.

### Optional secondary improvement

Increase the hardcoded channel buffer from `65536` to a larger value (e.g., `262144`) to absorb bursts better. Or make it configurable via `Keys.Checkpoints`.

## Files to modify

- `pkg/metricstore/walCheckpoint.go` — `walFileState` struct, `processMsg`, `flushAll`, and optionally channel buffer size

## Verification

1. `go build ./...` — compiles cleanly
2. `go test ./pkg/metricstore/...` — existing tests pass
3. Deploy and monitor: the warning should disappear or appear far less frequently


ExitPlanMode

Prompt: run go build and go test commands

# Fix WAL shard channel overflow (dropped messages)

## Context

The WAL staging goroutines can't keep up with constant-rate metric ingestion across multiple clusters, causing 7.2M+ messages to be dropped from WAL channels. Data is safe in memory (captured by next binary snapshot), but WAL crash-recovery coverage between snapshots is degraded.

## Root cause

The main loop in `WALStaging` (`walCheckpoint.go:266-296`) processes a batch of up to 257 messages, then calls `flushAll()` which flushes **every** open `bufio.Writer` — one write syscall per dirty host file.

With constant-rate ingestion across many hosts, most hosts are dirty in every batch. So each cycle is: 257 fast in-memory writes → hundreds of write syscalls to flush. Flush time dominates, the consumer can't sustain the ingestion rate, and the channel fills permanently (buffer size is irrelevant for sustained rate).

## Fix: Timer-based flushing

**Decouple flushing from message processing.** Instead of flushing after every batch, flush dirty files on a periodic timer (e.g., every 1 second).

### Changes in `pkg/metricstore/walCheckpoint.go`

1. **Add a `dirty` flag** to `walFileState` struct. Set it in `processMsg` after writing.

2. **Replace `flushAll()` with `flushDirty()`** that only flushes files where `dirty == true` and clears the flag.

3. **Restructure the main loop** to use a `time.Ticker` for flush scheduling:

```go
ticker := time.NewTicker(walFlushInterval) // e.g., 1s
defer ticker.Stop()

for {
    select {
    case <-ctx.Done():
        drain()
        return
    case msg, ok := <-msgCh:
        if !ok { return }
        processMsg(msg)
        // Drain without blocking — larger batch (4096)
        for range 4096 {
            select {
            case msg, ok := <-msgCh:
                if !ok { flushDirty(); return }
                processMsg(msg)
            case req := <-rotateCh:
                processRotate(req)
            default:
                goto drained
            }
        }
    drained:
        // No flush here — timer handles it
    case <-ticker.C:
        flushDirty()
    case req := <-rotateCh:
        processRotate(req)
    }
}
```

**Why this works**: Messages are processed at memory speed (`bufio.Writer` absorbs writes into its 4KB buffer). Flushes happen once per second instead of after every 257 messages, amortizing the syscall cost. The consumer throughput increases dramatically.

**Flush interval**: 1 second is a good default — short enough for crash safety, long enough to amortize I/O. Could be made configurable later if needed.

4. **Increase batch drain** from 256 to 4096 — with no flush at the end, the batch just controls how long before we re-check ctx/rotate channels.

## Files to modify

- `pkg/metricstore/walCheckpoint.go` — `walFileState` struct, main loop in `WALStaging`, `flushAll` → `flushDirty`

## Verification

1. `go build ./...`
2. `go test ./pkg/metricstore/...`
3. Deploy and monitor: the dropped-message warning should stop appearing


User has approved your plan. You can now start coding. Start with updating your todo list if applicable

Your plan has been saved to: /Users/jan/.claude/plans/floating-scribbling-raven.md
You can refer back to it if needed during implementation.

## Approved Plan:
# Fix WAL shard channel overflow (dropped messages)

## Context

The WAL staging goroutines can't keep up with constant-rate metric ingestion across multiple clusters, causing 7.2M+ messages to be dropped from WAL channels. Data is safe in memory (captured by next binary snapshot), but WAL crash-recovery coverage between snapshots is degraded.

## Root cause

The main loop in `WALStaging` (`walCheckpoint.go:266-296`) processes a batch of up to 257 messages, then calls `flushAll()` which flushes **every** open `bufio.Writer` — one write syscall per dirty host file.

With constant-rate ingestion across many hosts, most hosts are dirty in every batch. So each cycle is: 257 fast in-memory writes → hundreds of write syscalls to flush. Flush time dominates, the consumer can't sustain the ingestion rate, and the channel fills permanently (buffer size is irrelevant for sustained rate).

## Fix: Timer-based flushing

**Decouple flushing from message processing.** Instead of flushing after every batch, flush dirty files on a periodic timer (e.g., every 1 second).

### Changes in `pkg/metricstore/walCheckpoint.go`

1. **Add a `dirty` flag** to `walFileState` struct. Set it in `processMsg` after writing.

2. **Replace `flushAll()` with `flushDirty()`** that only flushes files where `dirty == true` and clears the flag.

3. **Restructure the main loop** to use a `time.Ticker` for flush scheduling:

```go
ticker := time.NewTicker(walFlushInterval) // e.g., 1s
defer ticker.Stop()

for {
    select {
    case <-ctx.Done():
        drain()
        return
    case msg, ok := <-msgCh:
        if !ok { return }
        processMsg(msg)
        // Drain without blocking — larger batch (4096)
        for range 4096 {
            select {
            case msg, ok := <-msgCh:
                if !ok { flushDirty(); return }
                processMsg(msg)
            case req := <-rotateCh:
                processRotate(req)
            default:
                goto drained
            }
        }
    drained:
        // No flush here — timer handles it
    case <-ticker.C:
        flushDirty()
    case req := <-rotateCh:
        processRotate(req)
    }
}
```

**Why this works**: Messages are processed at memory speed (`bufio.Writer` absorbs writes into its 4KB buffer). Flushes happen once per second instead of after every 257 messages, amortizing the syscall cost. The consumer throughput increases dramatically.

**Flush interval**: 1 second is a good default — short enough for crash safety, long enough to amortize I/O. Could be made configurable later if needed.

4. **Increase batch drain** from 256 to 4096 — with no flush at the end, the batch just controls how long before we re-check ctx/rotate channels.

## Files to modify

- `pkg/metricstore/walCheckpoint.go` — `walFileState` struct, main loop in `WALStaging`, `flushAll` → `flushDirty`

## Verification

1. `go build ./...`
2. `go test ./pkg/metricstore/...`
3. Deploy and monitor: the dropped-message warning should stop appearing


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
     7→//
     8→// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
     9→// per host, eliminating the extra memory overhead of the AvroStore and providing
    10→// truly continuous (per-write) crash safety.
    11→//
    12→// # Architecture
    13→//
    14→//	Metric write (DecodeLine)
    15→//	      │
    16→//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
    17→//	      │
    18→//	      └─► WALMessages channel
    19→//	               │
    20→//	               ▼

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼


    80→	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
    81→	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
    82→)
    83→
    84→// walShardChs holds per-shard channels for WAL messages.
    85→// Initialized by WALStaging; nil when WAL is not active.
    86→var walShardChs []chan *WALMessage
    87→
    88→// walShardRotateChs holds per-shard channels for WAL rotation requests.
    89→var walShardRotateChs []chan walRotateReq
    90→
    91→// walNumShards stores the number of shards (set during WALStaging init).
    92→var walNumShards int
    93→
    94→// WALMessage represents a single metric write to be appended to the WAL.
    95→// Cluster and Node are NOT stored in the WAL record (inferred from file path).
    96→type WALMessage struct {
    97→	MetricName string
    98→	Cluster    string
    99→	Node       string
   100→	Selector   []string
   101→	Value      schema.Float
   102→	Timestamp  int64
   103→}
   104→
   105→// walRotateReq requests WAL file rotation for a specific host directory.
   106→// The done channel is closed by the WAL goroutine when rotation is complete.
   107→type walRotateReq struct {
   108→	hostDir string
   109→	done    chan struct{}
   110→}
   111→
   112→// walFileState holds an open WAL file handle and buffered writer for one host directory.
   113→type walFileState struct {
   114→	f *os.File
   115→	w *bufio.Writer
   116→}
   117→
   118→// walShardIndex computes which shard a message belongs to based on cluster+node.
   119→// Uses FNV-1a hash for fast, well-distributed mapping.
   120→func walShardIndex(cluster, node string) int {
   121→	h := fnv.New32a()
   122→	h.Write([]byte(cluster))
   123→	h.Write([]byte{0})
   124→	h.Write([]byte(node))
   125→	return int(h.Sum32()) % walNumShards
   126→}
   127→
   128→// SendWALMessage routes a WAL message to the appropriate shard channel.
   129→// Returns false if the channel is full (message dropped).
   130→func SendWALMessage(msg *WALMessage) bool {
   131→	if walShardChs == nil {
   132→		return false
   133→	}
   134→	shard := walShardIndex(msg.Cluster, msg.Node)
   135→	select {
   136→	case walShardChs[shard] <- msg:
   137→		return true
   138→	default:
   139→		return false
   140→	}
   141→}
   142→
   143→// WALStaging starts N sharded background goroutines that receive WALMessage items
   144→// and append binary WAL records to per-host current.wal files.
   145→// Each shard owns a subset of hosts (determined by hash of cluster+node),
   146→// parallelizing serialization and I/O while keeping per-host writes serial.
   147→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   148→	if Keys.Checkpoints.FileFormat == "json" {
   149→		return

	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
)

// walShardChs holds per-shard channels for WAL messages.
// Initialized by WALStaging; nil when WAL is not active.
var walShardChs []chan *WALMessage

// walShardRotateChs holds per-shard channels for WAL rotation requests.
var walShardRotateChs []chan walRotateReq

// walNumShards stores the number of shards (set during WALStaging init).
var walNumShards int

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f *os.File
	w *bufio.Writer
}

// walShardIndex computes which shard a message belongs to based on cluster+node.
// Uses FNV-1a hash for fast, well-distributed mapping.
func walShardIndex(cluster, node string) int {
	h := fnv.New32a()
	h.Write([]byte(cluster))
	h.Write([]byte{0})
	h.Write([]byte(node))
	return int(h.Sum32()) % walNumShards
}

// SendWALMessage routes a WAL message to the appropriate shard channel.
// Returns false if the channel is full (message dropped).
func SendWALMessage(msg *WALMessage) bool {
	if walShardChs == nil {
		return false
	}
	shard := walShardIndex(msg.Cluster, msg.Node)
	select {
	case walShardChs[shard] <- msg:
		return true
	default:
		return false
	}
}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.
// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return


Now I have the full picture. Let me implement the changes.


Edit

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f *os.File
	w *bufio.Writer
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f     *os.File
	w     *bufio.Writer
	dirty bool
}

// walFlushInterval controls how often dirty WAL files are flushed to disk.
// Decoupling flushes from message processing lets the consumer run at memory
// speed, amortizing syscall overhead across many writes.
const walFlushInterval = 1 * time.Second


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go has been updated successfully.

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f     *os.File
	w     *bufio.Writer
	dirty bool
}

// walFlushInterval controls how often dirty WAL files are flushed to disk.
// Decoupling flushes from message processing lets the consumer run at memory
// speed, amortizing syscall overhead across many writes.
const walFlushInterval = 1 * time.Second

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
//
// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
// per host, eliminating the extra memory overhead of the AvroStore and providing
// truly continuous (per-write) crash safety.
//
// # Architecture
//
//	Metric write (DecodeLine)
//	      │
//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
//	      │
//	      └─► WALMessages channel
//	               │
//	               ▼
//	        WALStaging goroutine
//	               │
//	               ▼
//	        checkpoints/cluster/host/current.wal  (append-only, binary)
//
//	Periodic checkpoint (Checkpointing goroutine):
//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
//	  2. Signal WALStaging to truncate current.wal per host
//
//	On restart (FromCheckpoint):
//	  1. Load most recent <timestamp>.bin snapshot
//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
//
// # WAL Record Format
//
//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
//
//	payload:
//	  [8B timestamp int64]
//	  [2B metric_name_len uint16][N metric name bytes]
//	  [1B selector_count uint8]
//	  per selector: [1B selector_len uint8][M selector bytes]
//	  [4B value float32 bits]
//
// # Binary Snapshot Format
//
//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
//	Level tree (recursive):
//	  [4B num_metrics uint32]
//	  per metric:
//	    [2B name_len uint16][N name bytes]
//	    [8B frequency int64][8B start int64]
//	    [4B num_values uint32][num_values × 4B float32]
//	  [4B num_children uint32]
//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
package metricstore

import (
	"bufio"
	"context"
	"encoding/binary"
	"fmt"
	"hash/crc32"
	"hash/fnv"
	"io"
	"math"
	"os"
	"path"
	"strings"
	"sync"
	"sync/atomic"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Magic numbers for binary formats.
const (
	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
)

// walShardChs holds per-shard channels for WAL messages.
// Initialized by WALStaging; nil when WAL is not active.
var walShardChs []chan *WALMessage

// walShardRotateChs holds per-shard channels for WAL rotation requests.
var walShardRotateChs []chan walRotateReq

// walNumShards stores the number of shards (set during WALStaging init).
var walNumShards int

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f *os.File
	w *bufio.Writer
}

// walShardIndex computes which shard a message belongs to based on cluster+node.
// Uses FNV-1a hash for fast, well-distributed mapping.
func walShardIndex(cluster, node string) int {
	h := fnv.New32a()
	h.Write([]byte(cluster))
	h.Write([]byte{0})
	h.Write([]byte(node))
	return int(h.Sum32()) % walNumShards
}

// SendWALMessage routes a WAL message to the appropriate shard channel.
// Returns false if the channel is full (message dropped).
func SendWALMessage(msg *WALMessage) bool {
	if walShardChs == nil {
		return false
	}
	shard := walShardIndex(msg.Cluster, msg.Node)
	select {
	case walShardChs[shard] <- msg:
		return true
	default:
		return false
	}
}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.
// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return
	}

	walNumShards = max(Keys.NumWorkers, 1)
	chBufSize := max(65536/walNumShards, 1024)

	walShardChs = make([]chan *WALMessage, walNumShards)
	walShardRotateChs = make([]chan walRotateReq, walNumShards)

	for i := range walNumShards {
		walShardChs[i] = make(chan *WALMessage, chBufSize)
		walShardRotateChs[i] = make(chan walRotateReq, 64)
	}

	for i := range walNumShards {
		msgCh := walShardChs[i]
		rotateCh := walShardRotateChs[i]

		wg.Go(func() {
			hostFiles := make(map[string]*walFileState)

			defer func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
						ws.f.Close()
					}
				}
			}()

			getOrOpenWAL := func(hostDir string) *walFileState {
				ws, ok := hostFiles[hostDir]
				if ok {
					return ws
				}

				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
					return nil
				}

				walPath := path.Join(hostDir, "current.wal")
				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
					return nil
				}

				w := bufio.NewWriter(f)

				// Write file header magic if file is new (empty).
				info, err := f.Stat()
				if err == nil && info.Size() == 0 {
					var hdr [4]byte
					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
					if _, err := w.Write(hdr[:]); err != nil {
						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
						f.Close()
						return nil
					}
				}

				ws = &walFileState{f: f, w: w}
				hostFiles[hostDir] = ws
				return ws
			}

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}
				if err := writeWALRecordDirect(ws.w, msg); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
			}

			processRotate := func(req walRotateReq) {
				ws, ok := hostFiles[req.hostDir]
				if ok && ws.f != nil {
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(req.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, req.hostDir)
				}
				close(req.done)
			}

			flushAll := func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
					}
				}
			}

			drain := func() {
				for {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							return
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						flushAll()
						return
					}
				}
			}

			for {
				select {
				case <-ctx.Done():
					drain()
					return
				case msg, ok := <-msgCh:
					if !ok {
						return
					}
					processMsg(msg)

					// Drain up to 256 more messages without blocking to batch writes.
					for range 256 {
						select {
						case msg, ok := <-msgCh:
							if !ok {
								return
							}
							processMsg(msg)
						case req := <-rotateCh:
							processRotate(req)
						default:
							goto flushed
						}
					}
				flushed:
					flushAll()
				case req := <-rotateCh:
					processRotate(req)
				}
			}
		})
	}
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete. Each request is routed to the
// shard that owns the host directory.
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil {
		return
	}
	dones := make([]chan struct{}, len(hostDirs))
	for i, dir := range hostDirs {
		dones[i] = make(chan struct{})
		// Extract cluster/node from hostDir to find the right shard.
		// hostDir = rootDir/cluster/node
		shard := walShardIndexFromDir(dir)
		walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: dones[i]}
	}
	for _, done := range dones {
		<-done
	}
}

// walShardIndexFromDir extracts cluster and node from a host directory path
// and returns the shard index. The path format is: rootDir/cluster/node
func walShardIndexFromDir(hostDir string) int {
	// hostDir = .../cluster/node — extract last two path components
	node := path.Base(hostDir)
	cluster := path.Base(path.Dir(hostDir))
	return walShardIndex(cluster, node)
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete.
func RotateWALFilesAfterShutdown(hostDirs []string) {
	for _, dir := range hostDirs {
		walPath := path.Join(dir, "current.wal")
		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
		}
	}
}

// writeWALRecordDirect encodes a WAL record directly into the bufio.Writer,
// avoiding heap allocations by using a stack-allocated scratch buffer for
// the fixed-size header/trailer and computing CRC inline.
func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
	// Compute payload size.
	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
	for _, s := range msg.Selector {
		payloadSize += 1 + len(s)
	}

	// Write magic + payload length (8 bytes header).
	var hdr [8]byte
	binary.LittleEndian.PutUint32(hdr[0:4], walRecordMagic)
	binary.LittleEndian.PutUint32(hdr[4:8], uint32(payloadSize))
	if _, err := w.Write(hdr[:]); err != nil {
		return err
	}

	// We need to compute CRC over the payload as we write it.
	crc := crc32.NewIEEE()

	// Timestamp (8 bytes).
	var scratch [8]byte
	binary.LittleEndian.PutUint64(scratch[:8], uint64(msg.Timestamp))
	crc.Write(scratch[:8])
	if _, err := w.Write(scratch[:8]); err != nil {
		return err
	}

	// Metric name length (2 bytes) + metric name.
	binary.LittleEndian.PutUint16(scratch[:2], uint16(len(msg.MetricName)))
	crc.Write(scratch[:2])
	if _, err := w.Write(scratch[:2]); err != nil {
		return err
	}
	nameBytes := []byte(msg.MetricName)
	crc.Write(nameBytes)
	if _, err := w.Write(nameBytes); err != nil {
		return err
	}

	// Selector count (1 byte).
	scratch[0] = byte(len(msg.Selector))
	crc.Write(scratch[:1])
	if _, err := w.Write(scratch[:1]); err != nil {
		return err
	}

	// Selectors (1-byte length + bytes each).
	for _, sel := range msg.Selector {
		scratch[0] = byte(len(sel))
		crc.Write(scratch[:1])
		if _, err := w.Write(scratch[:1]); err != nil {
			return err
		}
		selBytes := []byte(sel)
		crc.Write(selBytes)
		if _, err := w.Write(selBytes); err != nil {
			return err
		}
	}

	// Value (4 bytes, float32 bits).
	binary.LittleEndian.PutUint32(scratch[:4], math.Float32bits(float32(msg.Value)))
	crc.Write(scratch[:4])
	if _, err := w.Write(scratch[:4]); err != nil {
		return err
	}

	// CRC32 (4 bytes).
	binary.LittleEndian.PutUint32(scratch[:4], crc.Sum32())
	_, err := w.Write(scratch[:4])
	return err
}

// buildWALPayload encodes a WALMessage into a binary payload (without magic/length/CRC).
func buildWALPayload(msg *WALMessage) []byte {
	size := 8 + 2 + len(msg.MetricName) + 1 + 4
	for _, s := range msg.Selector {
		size += 1 + len(s)
	}

	buf := make([]byte, 0, size)

	// Timestamp (8 bytes, little-endian int64)
	var ts [8]byte
	binary.LittleEndian.PutUint64(ts[:], uint64(msg.Timestamp))
	buf = append(buf, ts[:]...)

	// Metric name (2-byte length prefix + bytes)
	var mLen [2]byte
	binary.LittleEndian.PutUint16(mLen[:], uint16(len(msg.MetricName)))
	buf = append(buf, mLen[:]...)
	buf = append(buf, msg.MetricName...)

	// Selector count (1 byte)
	buf = append(buf, byte(len(msg.Selector)))

	// Selectors (1-byte length prefix + bytes each)
	for _, sel := range msg.Selector {
		buf = append(buf, byte(len(sel)))
		buf = append(buf, sel...)
	}

	// Value (4 bytes, float32 bit representation)
	var val [4]byte
	binary.LittleEndian.PutUint32(val[:], math.Float32bits(float32(msg.Value)))
	buf = append(buf, val[:]...)

	return buf
}

// writeWALRecord appends a binary WAL record to the writer.
// Format: [4B magic][4B payload_len][payload][4B CRC32]
func writeWALRecord(w io.Writer, msg *WALMessage) error {
	payload := buildWALPayload(msg)
	crc := crc32.ChecksumIEEE(payload)

	record := make([]byte, 0, 4+4+len(payload)+4)

	var magic [4]byte
	binary.LittleEndian.PutUint32(magic[:], walRecordMagic)
	record = append(record, magic[:]...)

	var pLen [4]byte
	binary.LittleEndian.PutUint32(pLen[:], uint32(len(payload)))
	record = append(record, pLen[:]...)

	record = append(record, payload...)

	var crcBytes [4]byte
	binary.LittleEndian.PutUint32(crcBytes[:], crc)
	record = append(record, crcBytes[:]...)

	_, err := w.Write(record)
	return err
}

// readWALRecord reads one WAL record from the reader.
// Returns (nil, nil) on clean EOF. Returns error on data corruption.
// A CRC mismatch indicates a truncated trailing record (expected on crash).
func readWALRecord(r io.Reader) (*WALMessage, error) {
	var magic uint32
	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
		if err == io.EOF {
			return nil, nil // Clean EOF
		}
		return nil, fmt.Errorf("read record magic: %w", err)
	}

	if magic != walRecordMagic {
		return nil, fmt.Errorf("invalid record magic 0x%08X (expected 0x%08X)", magic, walRecordMagic)
	}

	var payloadLen uint32
	if err := binary.Read(r, binary.LittleEndian, &payloadLen); err != nil {
		return nil, fmt.Errorf("read payload length: %w", err)
	}

	if payloadLen > 1<<20 { // 1 MB sanity limit
		return nil, fmt.Errorf("record payload too large: %d bytes", payloadLen)
	}

	payload := make([]byte, payloadLen)
	if _, err := io.ReadFull(r, payload); err != nil {
		return nil, fmt.Errorf("read payload: %w", err)
	}

	var storedCRC uint32
	if err := binary.Read(r, binary.LittleEndian, &storedCRC); err != nil {
		return nil, fmt.Errorf("read CRC: %w", err)
	}

	if crc32.ChecksumIEEE(payload) != storedCRC {
		return nil, fmt.Errorf("CRC mismatch (truncated write or corruption)")
	}

	return parseWALPayload(payload)
}

// parseWALPayload decodes a binary payload into a WALMessage.
func parseWALPayload(payload []byte) (*WALMessage, error) {
	if len(payload) < 8+2+1+4 {
		return nil, fmt.Errorf("payload too short: %d bytes", len(payload))
	}

	offset := 0

	// Timestamp (8 bytes)
	ts := int64(binary.LittleEndian.Uint64(payload[offset : offset+8]))
	offset += 8

	// Metric name (2-byte length + bytes)
	if offset+2 > len(payload) {
		return nil, fmt.Errorf("metric name length overflows payload")
	}
	mLen := int(binary.LittleEndian.Uint16(payload[offset : offset+2]))
	offset += 2

	if offset+mLen > len(payload) {
		return nil, fmt.Errorf("metric name overflows payload")
	}
	metricName := string(payload[offset : offset+mLen])
	offset += mLen

	// Selector count (1 byte)
	if offset >= len(payload) {
		return nil, fmt.Errorf("selector count overflows payload")
	}
	selCount := int(payload[offset])
	offset++

	selectors := make([]string, selCount)
	for i := range selCount {
		if offset >= len(payload) {
			return nil, fmt.Errorf("selector[%d] length overflows payload", i)
		}
		sLen := int(payload[offset])
		offset++

		if offset+sLen > len(payload) {
			return nil, fmt.Errorf("selector[%d] data overflows payload", i)
		}
		selectors[i] = string(payload[offset : offset+sLen])
		offset += sLen
	}

	// Value (4 bytes, float32 bits)
	if offset+4 > len(payload) {
		return nil, fmt.Errorf("value overflows payload")
	}
	bits := binary.LittleEndian.Uint32(payload[offset : offset+4])
	value := schema.Float(math.Float32frombits(bits))

	return &WALMessage{
		MetricName: metricName,
		Timestamp:  ts,
		Selector:   selectors,
		Value:      value,
	}, nil
}

// loadWALFile reads a WAL file and replays all valid records into the Level tree.
// l is the host-level node. Corrupt or partial trailing records are silently skipped
// (expected on crash). Records older than 'from' are skipped.
func (l *Level) loadWALFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	// Verify file header magic.
	var fileMagic uint32
	if err := binary.Read(br, binary.LittleEndian, &fileMagic); err != nil {
		if err == io.EOF {
			return nil // Empty file, no data
		}
		return fmt.Errorf("[METRICSTORE]> WAL: read file header: %w", err)
	}

	if fileMagic != walFileMagic {
		return fmt.Errorf("[METRICSTORE]> WAL: invalid file magic 0x%08X (expected 0x%08X)", fileMagic, walFileMagic)
	}

	// Cache level lookups to avoid repeated tree traversal.
	lvlCache := make(map[string]*Level)

	for {
		msg, err := readWALRecord(br)
		if err != nil {
			// Truncated trailing record is expected after a crash; stop replaying.
			cclog.Debugf("[METRICSTORE]> WAL: stopping replay at corrupted/partial record: %v", err)
			break
		}
		if msg == nil {
			break // Clean EOF
		}

		if msg.Timestamp < from {
			continue // Older than retention window
		}

		minfo, ok := m.Metrics[msg.MetricName]
		if !ok {
			continue // Unknown metric (config may have changed)
		}

		// Cache key is the null-separated selector path.
		cacheKey := joinSelector(msg.Selector)
		lvl, ok := lvlCache[cacheKey]
		if !ok {
			lvl = l.findLevelOrCreate(msg.Selector, len(m.Metrics))
			lvlCache[cacheKey] = lvl
		}

		// Write directly to the buffer, same as WriteToLevel but without the
		// global level lookup (we already have the right level).
		lvl.lock.Lock()
		b := lvl.metrics[minfo.offset]
		if b == nil {
			b = newBuffer(msg.Timestamp, minfo.Frequency)
			lvl.metrics[minfo.offset] = b
		}
		nb, writeErr := b.write(msg.Timestamp, msg.Value)
		if writeErr == nil && b != nb {
			lvl.metrics[minfo.offset] = nb
		}
		// Ignore write errors for timestamps before buffer start (can happen when
		// replaying WAL entries that predate a loaded snapshot's start time).
		lvl.lock.Unlock()
	}

	return nil
}

// joinSelector builds a cache key from a selector slice using null bytes as separators.
func joinSelector(sel []string) string {
	if len(sel) == 0 {
		return ""
	}
	var result strings.Builder
	result.WriteString(sel[0])
	for i := 1; i < len(sel); i++ {
		result.WriteString("\x00" + sel[i])
	}
	return result.String()
}

// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
// Returns the number of files written, the list of host directories that were
// successfully checkpointed (for WAL rotation), and any errors.
func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
	// Collect all cluster/host pairs.
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		hostDir  string
		selector []string
	}

	n, errs := int32(0), int32(0)
	var successDirs []string
	var successMu sync.Mutex

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)

	for range Keys.NumWorkers {
		go func() {
			defer wg.Done()
			for wi := range work {
				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
				if err != nil {
					if err == ErrNoNewArchiveData {
						continue
					}
					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
					successMu.Lock()
					successDirs = append(successDirs, wi.hostDir)
					successMu.Unlock()
				}
			}
		}()
	}

	for i := range levels {
		hostDir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			hostDir:  hostDir,
			selector: selectors[i],
		}
	}
	close(work)
	wg.Wait()

	if errs > 0 {
		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
	}
	return int(n), successDirs, nil
}

// toCheckpointBinary writes a binary snapshot file for a single host-level node.
// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}
	if cf == nil {
		return ErrNoNewArchiveData
	}

	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
		return fmt.Errorf("mkdir %s: %w", dir, err)
	}

	// Write to a temp file first, then rename (atomic on POSIX).
	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))

	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
	}

	bw := bufio.NewWriter(f)
	if err := writeBinarySnapshotFile(bw, cf); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return fmt.Errorf("write binary snapshot: %w", err)
	}
	if err := bw.Flush(); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return err
	}
	f.Close()

	return os.Rename(tmpPath, finalPath)
}

// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
		return err
	}
	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
		return err
	}
	return writeBinaryLevel(w, cf)
}

// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
		return err
	}

	for name, metric := range cf.Metrics {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
			return err
		}
		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
			return err
		}
		for _, v := range metric.Data {
			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
				return err
			}
		}
	}

	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
		return err
	}

	for name, child := range cf.Children {
		if err := writeString16(w, name); err != nil {
			return err
		}
		if err := writeBinaryLevel(w, child); err != nil {
			return err
		}
	}

	return nil
}

// writeString16 writes a 2-byte length-prefixed string to w.
func writeString16(w io.Writer, s string) error {
	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
		return err
	}
	_, err := io.WriteString(w, s)
	return err
}

// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
// The retention check (from) is applied to the file's 'to' timestamp.
func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)

	var magic uint32
	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
	}
	if magic != snapFileMagic {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
	}

	var fileFrom, fileTo int64
	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
	}
	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read to: %w", err)
	}

	if fileTo != 0 && fileTo < from {
		return nil // File is older than retention window, skip it
	}

	cf, err := readBinaryLevel(br)
	if err != nil {
		return fmt.Errorf("[METRICSTORE]> binary snapshot: read level tree: %w", err)
	}
	cf.From = fileFrom
	cf.To = fileTo

	return l.loadFile(cf, m)
}

// readBinaryLevel recursively reads a level from the binary snapshot format.
func readBinaryLevel(r io.Reader) (*CheckpointFile, error) {
	cf := &CheckpointFile{
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	var numMetrics uint32
	if err := binary.Read(r, binary.LittleEndian, &numMetrics); err != nil {
		return nil, fmt.Errorf("read num_metrics: %w", err)
	}

	for range numMetrics {
		name, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read metric name: %w", err)
		}

		var freq, start int64
		if err := binary.Read(r, binary.LittleEndian, &freq); err != nil {
			return nil, fmt.Errorf("read frequency for %s: %w", name, err)
		}
		if err := binary.Read(r, binary.LittleEndian, &start); err != nil {
			return nil, fmt.Errorf("read start for %s: %w", name, err)
		}

		var numValues uint32
		if err := binary.Read(r, binary.LittleEndian, &numValues); err != nil {
			return nil, fmt.Errorf("read num_values for %s: %w", name, err)
		}

		data := make([]schema.Float, numValues)
		for i := range numValues {
			var bits uint32
			if err := binary.Read(r, binary.LittleEndian, &bits); err != nil {
				return nil, fmt.Errorf("read value[%d] for %s: %w", i, name, err)
			}
			data[i] = schema.Float(math.Float32frombits(bits))
		}

		cf.Metrics[name] = &CheckpointMetrics{
			Frequency: freq,
			Start:     start,
			Data:      data,
		}
	}

	var numChildren uint32
	if err := binary.Read(r, binary.LittleEndian, &numChildren); err != nil {
		return nil, fmt.Errorf("read num_children: %w", err)
	}

	for range numChildren {
		childName, err := readString16(r)
		if err != nil {
			return nil, fmt.Errorf("read child name: %w", err)
		}

		child, err := readBinaryLevel(r)
		if err != nil {
			return nil, fmt.Errorf("read child %s: %w", childName, err)
		}
		cf.Children[childName] = child
	}

	return cf, nil
}

// readString16 reads a 2-byte length-prefixed string from r.
func readString16(r io.Reader) (string, error) {
	var sLen uint16
	if err := binary.Read(r, binary.LittleEndian, &sLen); err != nil {
		return "", err
	}
	buf := make([]byte, sLen)
	if _, err := io.ReadFull(r, buf); err != nil {
		return "", err
	}
	return string(buf), nil
}

+// walFlushInterval controls how often dirty WAL files are flushed to disk.

+// Decoupling flushes from message processing lets the consumer run at memory

+// speed, amortizing syscall overhead across many writes.

+const walFlushInterval = 1 * time.Second


Now update `processMsg` to set the dirty flag:


		processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}
				if err := writeWALRecordDirect(ws.w, msg); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
			}

		processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}
				if err := writeWALRecordDirect(ws.w, msg); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
				ws.dirty = true
			}