     1→# WAL Pipeline: Detailed Bug Analysis & Fix Strategies
     2→
     3→## BUG #1 & #2: PANIC ON SHUTDOWN (Send After Close)
     4→
     5→### Root Cause Analysis
     6→
     7→The shutdown sequence has a critical race window:
     8→
     9→```
    10→Timeline:
    11→T0:  Shutdown() called
    12→T1:  shutdownFunc() cancels context
    13→T2:  ReceiveNats() reads ctx.Done in select, begins exit
    14→T3:  ReceiveNats() calls wg.Wait() to drain worker goroutines
    15→T4:  NATS message still in-flight gets decoded by worker
    16→T5:  Worker calls DecodeLine() → SendWALMessage()
    17→T6:  Shutdown() AFTER ctx.Done, closes walShardChs[i]
    18→T7:  Worker at T5 reaches SendWALMessage, sends to CLOSED channel
    19→T8:  PANIC: send on closed channel
    20→```
    21→
    22→### The Problem in Detail
    23→
    24→File: `metricstore.go:273-314`
    25→
    26→```go
    27→func Shutdown() {
    28→    totalStart := time.Now()
    29→
    30→    shutdownFuncMu.Lock()
    31→    defer shutdownFuncMu.Unlock()
    32→    if shutdownFunc != nil {
    33→        shutdownFunc()  // <-- T1: Cancel context
    34→    }
    35→    cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))
    36→
    37→    // RACE WINDOW: ReceiveNats workers may still be decoding metrics!
    38→    if Keys.Checkpoints.FileFormat == "wal" {
    39→        for _, ch := range walShardChs {
    40→            close(ch)  // <-- T6: Close channel while T5 is executing!
    41→        }
    42→        // ...
    43→    }
    44→}
    45→```
    46→
    47→File: `lineprotocol.go:45-101`
    48→
    49→```go
    50→func ReceiveNats(ms *MemoryStore, workers int, ctx context.Context) error {
    51→    // ...
    52→    if workers > 1 {
    53→        wg.Add(workers)
    54→        for range workers {
    55→            go func() {
    56→                defer wg.Done()
    57→                for m := range msgs {  // <-- T4: Still receives message
    58→                    dec := lineprotocol.NewDecoderWithBytes(m)
    59→                    if err := DecodeLine(dec, ms, clusterTag); err != nil {
    60→                        // ...
    61→                    }
    62→                }
    63→            }()
    64→        }
    65→        nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
    66→            select {
    67→            case msgs <- data:  // <-- Message enqueued before ctx.Done
    68→            case <-ctx.Done():
    69→            }
    70→        })
    71→    }
    72→    go func() {
    73→        <-ctx.Done()
    74→        close(msgs)  // <-- Close msgs channel
    75→    }()
    76→    wg.Wait()  // <-- T3: Block here
    77→}
    78→```
    79→
    80→### Why It's Hard to Detect
    81→
    82→The `msgs` channel closes AFTER ctx.Done(), but there's a window where:
    83→1. A message is in the channel before ctx.Done
    84→2. A worker reads it AFTER ctx.Done
    85→3. The worker calls DecodeLine while Shutdown() closes walShardChs
    86→
    87→This is a **classic ordered shutdown failure**.
    88→
    89→### Why walShardChs != nil Check Doesn't Help
    90→
    91→File: `walCheckpoint.go:138-151`
    92→
    93→```go
    94→func SendWALMessage(msg *WALMessage) bool {
    95→    if walShardChs == nil {  // <-- Doesn't help!
    96→        return false
    97→    }
    98→    shard := walShardIndex(msg.Cluster, msg.Node)
    99→    select {
   100→    case walShardChs[shard] <- msg:  // <-- Still panics if channel is closed
   101→        return true
   102→    default:
   103→        return false
   104→    }
   105→}
   106→```
   107→
   108→**Go spec**: "A send on a closed channel proceeds by causing a run-time panic"
   109→- No check can prevent the panic
   110→- Only way: ensure channel is never closed while sends are in flight
   111→
   112→### Fix Strategy
   113→
   114→**Option A: Safe Shutdown Flag** (Recommended)
   115→
   116→```go
   117→// In walCheckpoint.go, global state:
   118→var walShuttingDown atomic.Bool
   119→
   120→// In SendWALMessage:
   121→func SendWALMessage(msg *WALMessage) bool {
   122→    if walShardChs == nil || walShuttingDown.Load() {
   123→        return false  // Return false instead of panicking
   124→    }
   125→    shard := walShardIndex(msg.Cluster, msg.Node)
   126→    select {
   127→    case walShardChs[shard] <- msg:
   128→        return true
   129→    default:
   130→        return false
   131→    }
   132→}
   133→
   134→// In Shutdown (metricstore.go):
   135→func Shutdown() {
   136→    if Keys.Checkpoints.FileFormat == "wal" {
   137→        walShuttingDown.Store(true)  // <-- Signal BEFORE closing
   138→        time.Sleep(100 * time.Millisecond)  // Grace period
   139→    }
   140→
   141→    shutdownFuncMu.Lock()
   142→    if shutdownFunc != nil {
   143→        shutdownFunc()  // Now cancel context
   144→    }
   145→    shutdownFuncMu.Unlock()
   146→
   147→    // Now safe to close channels
   148→    if Keys.Checkpoints.FileFormat == "wal" {
   149→        for _, ch := range walShardChs {
   150→            close(ch)
   151→        }
   152→        WaitForWALStagingDrain()
   153→    }
   154→
   155→    // Write checkpoint...
   156→}
   157→```
   158→
   159→**Option B: Graceful Drain** (More robust)
   160→
   161→```go
   162→// Add to walCheckpoint.go:
   163→func CloseWALChannels() {
   164→    if walShardChs == nil {
   165→        return
   166→    }
   167→    for _, ch := range walShardChs {
   168→        close(ch)
   169→    }
   170→}
   171→
   172→// In metricstore.go Shutdown():
   173→func Shutdown() {
   174→    // Phase 1: Request stop (but don't close channels yet)
   175→    shutdownFuncMu.Lock()
   176→    if shutdownFunc != nil {
   177→        shutdownFunc()  // Cancel context
   178→    }
   179→    shutdownFuncMu.Unlock()
   180→
   181→    cclog.Infof("[METRICSTORE]> Background workers cancelled")
   182→
   183→    // Phase 2: Grace period for in-flight metrics
   184→    if Keys.Checkpoints.FileFormat == "wal" {
   185→        time.Sleep(200 * time.Millisecond)
   186→
   187→        // Phase 3: Close producer channel (signals EOF)
   188→        CloseWALChannels()
   189→
   190→        // Phase 4: Wait for consumer to drain
   191→        WaitForWALStagingDrain()
   192→    }
   193→
   194→    // Phase 5: Final checkpoint
   195→    cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
   196→    // ...
   197→}
   198→```
   199→
   200→---
   201→
   202→## BUG #2: DIVISION BY ZERO ON STARTUP
   203→
   204→### Root Cause
   205→
   206→File: `walCheckpoint.go:162-171`
   207→
   208→```go
   209→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   210→    if Keys.Checkpoints.FileFormat == "json" {
   211→        return
   212→    }
   213→
   214→    walNumShards = max(Keys.NumWorkers, 1)  // <-- Line 162
   215→    // ...
   216→}
   217→```
   218→
   219→File: `metricstore.go:175, 188`
   220→
   221→```go
   222→// Init() execution order:
   223→WALStaging(wg, ctx)      // Line 175 - spawns goroutine
   224→// ...
   225→ReceiveNats(ms, Keys.NumWorkers, ctx)  // Line 188 - spawns workers
   226→```
   227→
   228→File: `lineprotocol.go:68-70`
   229→
   230→```go
   231→dec := lineprotocol.NewDecoderWithBytes(m)
   232→if err := DecodeLine(dec, ms, clusterTag); err != nil {  // Calls DecodeLine
   233→```
   234→
   235→File: `lineprotocol.go:355-364`
   236→
   237→```go
   238→func DecodeLine(...) error {
   239→    // ...
   240→    if Keys.Checkpoints.FileFormat == "wal" {
   241→        msg := &WALMessage{ ... }
   242→        if !SendWALMessage(msg) {  // Calls SendWALMessage
   243→```
   244→
   245→File: `walCheckpoint.go:130-136`
   246→
   247→```go
   248→func walShardIndex(cluster, node string) int {
   249→    h := fnv.New32a()
   250→    h.Write([]byte(cluster))
   251→    h.Write([]byte{0})
   252→    h.Write([]byte(node))
   253→    return int(h.Sum32()) % walNumShards  // <-- PANIC if walNumShards == 0
   254→}
   255→```
   256→
   257→### Why This Happens
   258→
   259→The goroutine spawned by `WALStaging()` doesn't execute immediately. Meanwhile:
   260→1. Init() continues to line 188
   261→2. Calls ReceiveNats()
   262→3. Calls nc.Subscribe() (subscribes to NATS subject)
   263→4. Before WALStaging goroutine can set walNumShards, NATS delivers a message
   264→5. Calls DecodeLine → SendWALMessage → walShardIndex
   265→6. **PANIC**: modulo by zero
   266→
   267→### Fix Strategy
   268→
   269→**Synchronous Initialization** (Recommended)
   270→
   271→```go
   272→// In walCheckpoint.go:
   273→func InitWALShards(numShards int) {
   274→    walNumShards = max(numShards, 1)
   275→    chBufSize := max(65536/walNumShards, 1024)
   276→
   277→    walShardChs = make([]chan *WALMessage, walNumShards)
   278→    walShardRotateChs = make([]chan walRotateReq, walNumShards)
   279→
   280→    for i := range walNumShards {
   281→        walShardChs[i] = make(chan *WALMessage, chBufSize)
   282→        walShardRotateChs[i] = make(chan walRotateReq, 64)
   283→    }
   284→}
   285→
   286→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   287→    if Keys.Checkpoints.FileFormat == "json" {
   288→        return
   289→    }
   290→
   291→    // Channels already initialized by InitWALShards()
   292→
   293→    for i := range walNumShards {
   294→        // ... spawn goroutine for shard i
   295→    }
   296→}
   297→
   298→// In metricstore.go Init():
   299→// Synchronously initialize shards BEFORE starting any workers
   300→if Keys.Checkpoints.FileFormat == "wal" {
   301→    InitWALShards(Keys.NumWorkers)
   302→}
   303→
   304→// Now safe to start workers:
   305→WALStaging(wg, ctx)
   306→// ...
   307→ReceiveNats(ms, Keys.NumWorkers, ctx)  // Safe, walNumShards is set
   308→```
   309→
   310→---
   311→
   312→## BUG #3: ROTATION DEADLOCK
   313→
   314→### Root Cause
   315→
   316→File: `walCheckpoint.go:87-90`
   317→
   318→```go
   319→var walShardChs []chan *WALMessage
   320→var walShardRotateChs []chan walRotateReq  // <-- NEVER closed
   321→```
   322→
   323→Timeline:
   324→```
   325→T0: Shutdown() called
   326→T1: ctx.Done() → WALStaging goroutine sees EOF
   327→T2: WALStaging reads msgCh, ok=false (channel closed)
   328→T3: WALStaging returns, goroutine exits
   329→T4: Meanwhile, Checkpointing calls RotateWALFiles(hostDirs)
   330→T5: RotateWALFiles sends to rotateCh of now-exited goroutine
   331→T6: RotateWALFiles blocks forever on <-done channel
   332→T7: Shutdown hangs waiting for checkpoint
   333→```
   334→
   335→### Fix Strategy
   336→
   337→**Option A: Close rotation channels before exit**
   338→
   339→```go
   340→// In walCheckpoint.go, add:
   341→var walShardRotateChsMu sync.Mutex
   342→
   343→func CloseRotationChannels() {
   344→    walShardRotateChsMu.Lock()
   345→    defer walShardRotateChsMu.Unlock()
   346→
   347→    if walShardRotateChs == nil {
   348→        return
   349→    }
   350→    for _, ch := range walShardRotateChs {
   351→        close(ch)
   352→    }
   353→    walShardRotateChs = nil
   354→}
   355→
   356→// In WALStaging goroutine exit path:
   357→defer func() {
   358→    // ... existing cleanup ...
   359→    if isLastShardGoroutine {  // Or: use waitgroup
   360→        CloseRotationChannels()
   361→    }
   362→}()
   363→```
   364→
   365→**Option B: Check for closed channel in RotateWALFiles** (Simpler)
   366→
   367→```go
   368→// In RotateWALFiles:
   369→func RotateWALFiles(hostDirs []string) {
   370→    if walShardRotateChs == nil {
   371→        return
   372→    }
   373→
   374→    dones := make([]chan struct{}, len(hostDirs))
   375→    timeouts := make([]<-chan time.Time, len(hostDirs))
   376→
   377→    for i, dir := range hostDirs {
   378→        dones[i] = make(chan struct{})
   379→        shard := walShardIndexFromDir(dir)
   380→
   381→        // Use timeout to detect if goroutine is dead
   382→        timeouts[i] = time.After(5 * time.Second)
   383→
   384→        select {
   385→        case walShardRotateChs[shard] <- walRotateReq{
   386→            hostDir: dir,
   387→            done:    dones[i],
   388→        }:
   389→            // Sent successfully
   390→        case <-timeouts[i]:
   391→            // Goroutine not responding, close the done channel ourselves
   392→            close(dones[i])
   393→        }
   394→    }
   395→
   396→    // Wait for responses with timeout
   397→    for i, done := range dones {
   398→        select {
   399→        case <-done:
   400→            // Normal completion
   401→        case <-timeouts[i]:
   402→            // Timeout, already handled above
   403→            cclog.Warnf("[METRICSTORE]> Rotation timeout for host %d", i)
   404→        }
   405→    }
   406→}
   407→```
   408→
   409→**Option C: Rotate before shutdown** (Best)
   410→
   411→Don't call RotateWALFiles during shutdown at all. The final checkpoint doesn't need WAL rotation.
   412→
   413→```go
   414→// In metricstore.go Shutdown():
   415→if Keys.Checkpoints.FileFormat == "wal" {
   416→    // Don't call RotateWALFiles here
   417→    // Files are deleted in ToCheckpointWAL workers anyway
   418→
   419→    files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   420→    // ToCheckpointWAL deletes current.wal files directly
   421→}
   422→```
   423→
   424→---
   425→
   426→## BUG #4: SILENT DATA LOSS ON BACKPRESSURE
   427→
   428→### The Issue
   429→
   430→```
   431→Channel size = 65536 / walNumShards
   432→With 4 workers: 65536 / 4 = 16384 per shard
   433→At 100k msg/sec across 4 workers: 25k msg/sec per shard
   434→Channel fills in: 16384 / 25000 = 0.65 seconds
   435→```
   436→
   437→Every drop violates the durability guarantee, but drops are only logged every 10k drops.
   438→
   439→### Fix Strategy: Implement Real Backpressure
   440→
   441→```go
   442→// In lineprotocol.go DecodeLine:
   443→if Keys.Checkpoints.FileFormat == "wal" {
   444→    msg := &WALMessage{
   445→        MetricName: string(st.metricBuf),
   446→        Cluster:    cluster,
   447→        Node:       host,
   448→        Selector:   append([]string{}, st.selector...),
   449→        Value:      metric.Value,
   450→        Timestamp:  time.Unix(),
   451→    }
   452→
   453→    // Option 1: Block on full (applies backpressure)
   454→    if !SendWALMessageBlocking(msg, 1*time.Second) {
   455→        cclog.Errorf("[METRICSTORE]> WAL timeout for %s/%s metric %s, dropping",
   456→            cluster, host, string(st.metricBuf))
   457→        // Now we know it was dropped, and metrics show it
   458→    }
   459→
   460→    // Option 2: Increase channel size dynamically
   461→    // or use unbounded queue
   462→}
   463→
   464→// In walCheckpoint.go:
   465→func SendWALMessageBlocking(msg *WALMessage, timeout time.Duration) bool {
   466→    if walShardChs == nil {
   467→        return false
   468→    }
   469→    shard := walShardIndex(msg.Cluster, msg.Node)
   470→
   471→    select {
   472→    case walShardChs[shard] <- msg:
   473→        return true
   474→    case <-time.After(timeout):
   475→        return false  // Timeout, metric dropped (with explicit error logged)
   476→    }
   477→}
   478→
   479→// Or: Increase channel buffer significantly
   480→const chBufSize = 1 << 20  // 1 million messages, ~100MB
   481→```
   482→
   483→---
   484→
   485→## BUG #5: LOST METRICS AT SHUTDOWN
   486→
   487→### The Issue
   488→
   489→```go
   490→case <-ctx.Done():
   491→    // Comment says: "buffered messages are redundant"
   492→    // WRONG! If metric arrives between snapshot T1 and ctx.Done at T2,
   493→    // it's in the channel but won't be persisted.
   494→    flushDirty()
   495→    return
   496→```
   497→
   498→### Fix Strategy: Grace Period
   499→
   500→```go
   501→// In walCheckpoint.go WALStaging:
   502→const ShutdownGracePeriod = 500 * time.Millisecond
   503→
   504→for {
   505→    select {
   506→    case <-ctx.Done():
   507→        // Don't exit immediately. Drain all buffered messages
   508→        // that arrived before ctx.Done was signaled.
   509→
   510→        // First, process any messages already in the channel
   511→        for {
   512→            select {
   513→            case msg, ok := <-msgCh:
   514→                if !ok {
   515→                    // Channel closed, now safe to exit
   516→                    flushDirty()
   517→                    return
   518→                }
   519→                processMsg(msg)
   520→            case req := <-rotateCh:
   521→                processRotate(req)
   522→            default:
   523→                // No more messages immediately available
   524→                // Give grace period for in-flight messages
   525→                select {
   526→                case msg, ok := <-msgCh:
   527→                    if !ok {
   528→                        flushDirty()
   529→                        return
   530→                    }
   531→                    processMsg(msg)
   532→                case <-time.After(ShutdownGracePeriod):
   533→                    // Grace period expired, exit
   534→                    flushDirty()
   535→                    return
   536→                }
   537→            }
   538→        }
   539→    // ... other cases
   540→    }
   541→}
   542→```
   543→
   544→---
   545→
   546→## BUG #6: PARTIAL RECORD ON DISK FULL
   547→
   548→### The Issue
   549→
   550→```go
   551→func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
   552→    var hdr [8]byte
   553→    binary.LittleEndian.PutUint32(hdr[0:4], walRecordMagic)
   554→    binary.LittleEndian.PutUint32(hdr[4:8], uint32(payloadSize))
   555→    if _, err := w.Write(hdr[:]); err != nil {
   556→        return err  // <-- Partial data in buffer!
   557→    }
   558→    // ... more writes ...
   559→}
   560→```
   561→
   562→If any write fails, previous writes sit in bufio.Writer. Next call appends, corrupting the file.
   563→
   564→### Fix Strategy: Two-Phase Write
   565→
   566→```go
   567→func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
   568→    // Phase 1: Prepare all data in a buffer
   569→    var buf bytes.Buffer
   570→
   571→    // Write to temporary buffer
   572→    payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
   573→    for _, s := range msg.Selector {
   574→        payloadSize += 1 + len(s)
   575→    }
   576→
   577→    // Write magic + length
   578→    var hdr [8]byte
   579→    binary.LittleEndian.PutUint32(hdr[0:4], walRecordMagic)
   580→    binary.LittleEndian.PutUint32(hdr[4:8], uint32(payloadSize))
   581→    buf.Write(hdr[:])
   582→
   583→    // ... prepare payload in buf ...
   584→
   585→    // Phase 2: Compute CRC
   586→    crc := crc32.NewIEEE()
   587→    payload := buf.Bytes()[8:]  // Skip magic + length
   588→    crc.Write(payload)
   589→
   590→    // Phase 3: Atomic write (buffer is complete, write as one unit)
   591→    if _, err := w.Write(buf.Bytes()); err != nil {
   592→        return err
   593→    }
   594→
   595→    var crcBuf [4]byte
   596→    binary.LittleEndian.PutUint32(crcBuf[:], crc.Sum32())
   597→    if _, err := w.Write(crcBuf[:]); err != nil {
   598→        return err
   599→    }
   600→
   601→    return nil
   602→}
   603→```
   604→
   605→---
   606→
   607→## BUG #8: NON-ATOMIC SNAPSHOT + WAL ROTATION
   608→
   609→### The Issue
   610→
   611→```
   612→T1: Snapshot writes data up to timestamp T0
   613→T2: Worker deletes current.wal
   614→T3: Metric arrives with timestamp T0+100ms
   615→T4: Metric written to current.wal
   616→T5: Process crashes
   617→T6: On restart: Snapshot from T0 + empty WAL
   618→T7: Data from T0+100ms is lost!
   619→```
   620→
   621→### Fix Strategy: Atomic Rotation
   622→
   623→**Option A: Rename + Replace**
   624→
   625→```go
   626→// In ToCheckpointWAL worker (line 697):
   627→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   628→    // ... write snapshot ...
   629→
   630→    // Atomic rotation: rename current.wal to timestamp.wal.old
   631→    walPath := path.Join(dir, "current.wal")
   632→    oldPath := path.Join(dir, fmt.Sprintf("%d.wal.old", from))
   633→
   634→    if err := os.Rename(walPath, oldPath); err != nil && !os.IsNotExist(err) {
   635→        return fmt.Errorf("rename WAL: %w", err)
   636→    }
   637→
   638→    // old file will be deleted by cleanup job
   639→    // or: delete it here if we're sure no reads are in progress
   640→    os.Remove(oldPath)
   641→
   642→    return nil
   643→}
   644→```
   645→
   646→**Option B: Timestamp Marker**
   647→
   648→```go
   649→// Write checkpoint metadata with timestamp
   650→type CheckpointMetadata struct {
   651→    SnapshotTS int64  // Snapshot created up to this time
   652→    CreatedAt  time.Time
   653→}
   654→
   655→// Store in .json or separate metadata file
   656→
   657→// On load: Only replay WAL entries > SnapshotTS
   658→```
   659→
   660→**Option C: WAL Truncation Marker**
   661→
   662→```go
   663→// In current.wal, write a special marker record:
   664→const walTruncateMarker = uint32(0xCC1DA7FF)
   665→
   666→func writeWALTruncateMarker(w *bufio.Writer, ts int64) error {
   667→    var hdr [12]byte
   668→    binary.LittleEndian.PutUint32(hdr[0:4], walTruncateMarker)
   669→    binary.LittleEndian.PutUint32(hdr[4:8], 8)
   670→    binary.LittleEndian.PutUint64(hdr[8:16], uint64(ts))
   671→
   672→    _, err := w.Write(hdr[:])
   673→    return err
   674→}
   675→
   676→// On load: Stop replaying at truncate marker
   677→```
   678→
   679→---
   680→
   681→## BUG #9: INITIALIZATION RACE
   682→
   683→Same as BUG #2 analysis. Initialize walNumShards synchronously before NATS subscription.
   684→
   685→---
   686→
   687→## TESTING CHECKLIST
   688→
   689→```go
   690→// Test cases needed:
   691→func TestShutdownWithInFlightMetrics(t *testing.T) {
   692→    // Send metric, immediately shutdown
   693→    // Verify no panic, metric persisted or acknowledged dropped
   694→}
   695→
   696→func TestRotationDeadlock(t *testing.T) {
   697→    // Trigger checkpoint rotation during heavy write
   698→    // Verify shutdown completes within timeout
   699→}
   700→
   701→func TestDiskFullRecovery(t *testing.T) {
   702→    // Simulate disk full during WAL write
   703→    // Verify WAL is not corrupted on restart
   704→}
   705→
   706→func TestNATSMetricsWithWALDisabled(t *testing.T) {
   707→    // Run with WAL disabled
   708→    // Verify no crashes
   709→}
   710→
   711→func TestBackpressure(t *testing.T) {
   712→    // Send metrics faster than WAL can write
   713→    // Verify either blocked or acknowledged dropped (not silent)
   714→}
   715→
   716→func TestSnapshotWALConsistency(t *testing.T) {
   717→    // Write snapshot, then WAL, crash during gap
   718→    // Verify data completeness on restart
   719→}
   720→```
   721→
   722→

# WAL Pipeline: Detailed Bug Analysis & Fix Strategies

## BUG #1 & #2: PANIC ON SHUTDOWN (Send After Close)

### Root Cause Analysis

The shutdown sequence has a critical race window:

```
Timeline:
T0:  Shutdown() called
T1:  shutdownFunc() cancels context
T2:  ReceiveNats() reads ctx.Done in select, begins exit
T3:  ReceiveNats() calls wg.Wait() to drain worker goroutines
T4:  NATS message still in-flight gets decoded by worker
T5:  Worker calls DecodeLine() → SendWALMessage()
T6:  Shutdown() AFTER ctx.Done, closes walShardChs[i]
T7:  Worker at T5 reaches SendWALMessage, sends to CLOSED channel
T8:  PANIC: send on closed channel
```

### The Problem in Detail

File: `metricstore.go:273-314`

```go
func Shutdown() {
    totalStart := time.Now()

    shutdownFuncMu.Lock()
    defer shutdownFuncMu.Unlock()
    if shutdownFunc != nil {
        shutdownFunc()  // <-- T1: Cancel context
    }
    cclog.Infof("[METRICSTORE]> Background workers cancelled (%v)", time.Since(totalStart))

    // RACE WINDOW: ReceiveNats workers may still be decoding metrics!
    if Keys.Checkpoints.FileFormat == "wal" {
        for _, ch := range walShardChs {
            close(ch)  // <-- T6: Close channel while T5 is executing!
        }
        // ...
    }
}
```

File: `lineprotocol.go:45-101`

```go
func ReceiveNats(ms *MemoryStore, workers int, ctx context.Context) error {
    // ...
    if workers > 1 {
        wg.Add(workers)
        for range workers {
            go func() {
                defer wg.Done()
                for m := range msgs {  // <-- T4: Still receives message
                    dec := lineprotocol.NewDecoderWithBytes(m)
                    if err := DecodeLine(dec, ms, clusterTag); err != nil {
                        // ...
                    }
                }
            }()
        }
        nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
            select {
            case msgs <- data:  // <-- Message enqueued before ctx.Done
            case <-ctx.Done():
            }
        })
    }
    go func() {
        <-ctx.Done()
        close(msgs)  // <-- Close msgs channel
    }()
    wg.Wait()  // <-- T3: Block here
}
```

### Why It's Hard to Detect

The `msgs` channel closes AFTER ctx.Done(), but there's a window where:
1. A message is in the channel before ctx.Done
2. A worker reads it AFTER ctx.Done
3. The worker calls DecodeLine while Shutdown() closes walShardChs

This is a **classic ordered shutdown failure**.

### Why walShardChs != nil Check Doesn't Help

File: `walCheckpoint.go:138-151`

```go
func SendWALMessage(msg *WALMessage) bool {
    if walShardChs == nil {  // <-- Doesn't help!
        return false
    }
    shard := walShardIndex(msg.Cluster, msg.Node)
    select {
    case walShardChs[shard] <- msg:  // <-- Still panics if channel is closed
        return true
    default:
        return false
    }
}
```

**Go spec**: "A send on a closed channel proceeds by causing a run-time panic"
- No check can prevent the panic
- Only way: ensure channel is never closed while sends are in flight

### Fix Strategy

**Option A: Safe Shutdown Flag** (Recommended)

```go
// In walCheckpoint.go, global state:
var walShuttingDown atomic.Bool

// In SendWALMessage:
func SendWALMessage(msg *WALMessage) bool {
    if walShardChs == nil || walShuttingDown.Load() {
        return false  // Return false instead of panicking
    }
    shard := walShardIndex(msg.Cluster, msg.Node)
    select {
    case walShardChs[shard] <- msg:
        return true
    default:
        return false
    }
}

// In Shutdown (metricstore.go):
func Shutdown() {
    if Keys.Checkpoints.FileFormat == "wal" {
        walShuttingDown.Store(true)  // <-- Signal BEFORE closing
        time.Sleep(100 * time.Millisecond)  // Grace period
    }

    shutdownFuncMu.Lock()
    if shutdownFunc != nil {
        shutdownFunc()  // Now cancel context
    }
    shutdownFuncMu.Unlock()

    // Now safe to close channels
    if Keys.Checkpoints.FileFormat == "wal" {
        for _, ch := range walShardChs {
            close(ch)
        }
        WaitForWALStagingDrain()
    }

    // Write checkpoint...
}
```

**Option B: Graceful Drain** (More robust)

```go
// Add to walCheckpoint.go:
func CloseWALChannels() {
    if walShardChs == nil {
        return
    }
    for _, ch := range walShardChs {
        close(ch)
    }
}

// In metricstore.go Shutdown():
func Shutdown() {
    // Phase 1: Request stop (but don't close channels yet)
    shutdownFuncMu.Lock()
    if shutdownFunc != nil {
        shutdownFunc()  // Cancel context
    }
    shutdownFuncMu.Unlock()

    cclog.Infof("[METRICSTORE]> Background workers cancelled")

    // Phase 2: Grace period for in-flight metrics
    if Keys.Checkpoints.FileFormat == "wal" {
        time.Sleep(200 * time.Millisecond)

        // Phase 3: Close producer channel (signals EOF)
        CloseWALChannels()

        // Phase 4: Wait for consumer to drain
        WaitForWALStagingDrain()
    }

    // Phase 5: Final checkpoint
    cclog.Infof("[METRICSTORE]> Writing checkpoint to '%s'...", Keys.Checkpoints.RootDir)
    // ...
}
```

---

## BUG #2: DIVISION BY ZERO ON STARTUP

### Root Cause

File: `walCheckpoint.go:162-171`

```go
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
    if Keys.Checkpoints.FileFormat == "json" {
        return
    }

    walNumShards = max(Keys.NumWorkers, 1)  // <-- Line 162
    // ...
}
```

File: `metricstore.go:175, 188`

```go
// Init() execution order:
WALStaging(wg, ctx)      // Line 175 - spawns goroutine
// ...
ReceiveNats(ms, Keys.NumWorkers, ctx)  // Line 188 - spawns workers
```

File: `lineprotocol.go:68-70`

```go
dec := lineprotocol.NewDecoderWithBytes(m)
if err := DecodeLine(dec, ms, clusterTag); err != nil {  // Calls DecodeLine
```

File: `lineprotocol.go:355-364`

```go
func DecodeLine(...) error {
    // ...
    if Keys.Checkpoints.FileFormat == "wal" {
        msg := &WALMessage{ ... }
        if !SendWALMessage(msg) {  // Calls SendWALMessage
```

File: `walCheckpoint.go:130-136`

```go
func walShardIndex(cluster, node string) int {
    h := fnv.New32a()
    h.Write([]byte(cluster))
    h.Write([]byte{0})
    h.Write([]byte(node))
    return int(h.Sum32()) % walNumShards  // <-- PANIC if walNumShards == 0
}
```

### Why This Happens

The goroutine spawned by `WALStaging()` doesn't execute immediately. Meanwhile:
1. Init() continues to line 188
2. Calls ReceiveNats()
3. Calls nc.Subscribe() (subscribes to NATS subject)
4. Before WALStaging goroutine can set walNumShards, NATS delivers a message
5. Calls DecodeLine → SendWALMessage → walShardIndex
6. **PANIC**: modulo by zero

### Fix Strategy

**Synchronous Initialization** (Recommended)

```go
// In walCheckpoint.go:
func InitWALShards(numShards int) {
    walNumShards = max(numShards, 1)
    chBufSize := max(65536/walNumShards, 1024)

    walShardChs = make([]chan *WALMessage, walNumShards)
    walShardRotateChs = make([]chan walRotateReq, walNumShards)

    for i := range walNumShards {
        walShardChs[i] = make(chan *WALMessage, chBufSize)
        walShardRotateChs[i] = make(chan walRotateReq, 64)
    }
}

func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
    if Keys.Checkpoints.FileFormat == "json" {
        return
    }

    // Channels already initialized by InitWALShards()

    for i := range walNumShards {
        // ... spawn goroutine for shard i
    }
}

// In metricstore.go Init():
// Synchronously initialize shards BEFORE starting any workers
if Keys.Checkpoints.FileFormat == "wal" {
    InitWALShards(Keys.NumWorkers)
}

// Now safe to start workers:
WALStaging(wg, ctx)
// ...
ReceiveNats(ms, Keys.NumWorkers, ctx)  // Safe, walNumShards is set
```

---

## BUG #3: ROTATION DEADLOCK

### Root Cause

File: `walCheckpoint.go:87-90`

```go
var walShardChs []chan *WALMessage
var walShardRotateChs []chan walRotateReq  // <-- NEVER closed
```

Timeline:
```
T0: Shutdown() called
T1: ctx.Done() → WALStaging goroutine sees EOF
T2: WALStaging reads msgCh, ok=false (channel closed)
T3: WALStaging returns, goroutine exits
T4: Meanwhile, Checkpointing calls RotateWALFiles(hostDirs)
T5: RotateWALFiles sends to rotateCh of now-exited goroutine
T6: RotateWALFiles blocks forever on <-done channel
T7: Shutdown hangs waiting for checkpoint
```

### Fix Strategy

**Option A: Close rotation channels before exit**

```go
// In walCheckpoint.go, add:
var walShardRotateChsMu sync.Mutex

func CloseRotationChannels() {
    walShardRotateChsMu.Lock()
    defer walShardRotateChsMu.Unlock()

    if walShardRotateChs == nil {
        return
    }
    for _, ch := range walShardRotateChs {
        close(ch)
    }
    walShardRotateChs = nil
}

// In WALStaging goroutine exit path:
defer func() {
    // ... existing cleanup ...
    if isLastShardGoroutine {  // Or: use waitgroup
        CloseRotationChannels()
    }
}()
```

**Option B: Check for closed channel in RotateWALFiles** (Simpler)

```go
// In RotateWALFiles:
func RotateWALFiles(hostDirs []string) {
    if walShardRotateChs == nil {
        return
    }

    dones := make([]chan struct{}, len(hostDirs))
    timeouts := make([]<-chan time.Time, len(hostDirs))

    for i, dir := range hostDirs {
        dones[i] = make(chan struct{})
        shard := walShardIndexFromDir(dir)

        // Use timeout to detect if goroutine is dead
        timeouts[i] = time.After(5 * time.Second)

        select {
        case walShardRotateChs[shard] <- walRotateReq{
            hostDir: dir,
            done:    dones[i],
        }:
            // Sent successfully
        case <-timeouts[i]:
            // Goroutine not responding, close the done channel ourselves
            close(dones[i])
        }
    }

    // Wait for responses with timeout
    for i, done := range dones {
        select {
        case <-done:
            // Normal completion
        case <-timeouts[i]:
            // Timeout, already handled above
            cclog.Warnf("[METRICSTORE]> Rotation timeout for host %d", i)
        }
    }
}
```

**Option C: Rotate before shutdown** (Best)

Don't call RotateWALFiles during shutdown at all. The final checkpoint doesn't need WAL rotation.

```go
// In metricstore.go Shutdown():
if Keys.Checkpoints.FileFormat == "wal" {
    // Don't call RotateWALFiles here
    // Files are deleted in ToCheckpointWAL workers anyway

    files, _, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
    // ToCheckpointWAL deletes current.wal files directly
}
```

---

## BUG #4: SILENT DATA LOSS ON BACKPRESSURE

### The Issue

```
Channel size = 65536 / walNumShards
With 4 workers: 65536 / 4 = 16384 per shard
At 100k msg/sec across 4 workers: 25k msg/sec per shard
Channel fills in: 16384 / 25000 = 0.65 seconds
```

Every drop violates the durability guarantee, but drops are only logged every 10k drops.

### Fix Strategy: Implement Real Backpressure

```go
// In lineprotocol.go DecodeLine:
if Keys.Checkpoints.FileFormat == "wal" {
    msg := &WALMessage{
        MetricName: string(st.metricBuf),
        Cluster:    cluster,
        Node:       host,
        Selector:   append([]string{}, st.selector...),
        Value:      metric.Value,
        Timestamp:  time.Unix(),
    }

    // Option 1: Block on full (applies backpressure)
    if !SendWALMessageBlocking(msg, 1*time.Second) {
        cclog.Errorf("[METRICSTORE]> WAL timeout for %s/%s metric %s, dropping",
            cluster, host, string(st.metricBuf))
        // Now we know it was dropped, and metrics show it
    }

    // Option 2: Increase channel size dynamically
    // or use unbounded queue
}

// In walCheckpoint.go:
func SendWALMessageBlocking(msg *WALMessage, timeout time.Duration) bool {
    if walShardChs == nil {
        return false
    }
    shard := walShardIndex(msg.Cluster, msg.Node)

    select {
    case walShardChs[shard] <- msg:
        return true
    case <-time.After(timeout):
        return false  // Timeout, metric dropped (with explicit error logged)
    }
}

// Or: Increase channel buffer significantly
const chBufSize = 1 << 20  // 1 million messages, ~100MB
```

---

## BUG #5: LOST METRICS AT SHUTDOWN

### The Issue

```go
case <-ctx.Done():
    // Comment says: "buffered messages are redundant"
    // WRONG! If metric arrives between snapshot T1 and ctx.Done at T2,
    // it's in the channel but won't be persisted.
    flushDirty()
    return
```

### Fix Strategy: Grace Period

```go
// In walCheckpoint.go WALStaging:
const ShutdownGracePeriod = 500 * time.Millisecond

for {
    select {
    case <-ctx.Done():
        // Don't exit immediately. Drain all buffered messages
        // that arrived before ctx.Done was signaled.

        // First, process any messages already in the channel
        for {
            select {
            case msg, ok := <-msgCh:
                if !ok {
                    // Channel closed, now safe to exit
                    flushDirty()
                    return
                }
                processMsg(msg)
            case req := <-rotateCh:
                processRotate(req)
            default:
                // No more messages immediately available
                // Give grace period for in-flight messages
                select {
                case msg, ok := <-msgCh:
                    if !ok {
                        flushDirty()
                        return
                    }
                    processMsg(msg)
                case <-time.After(ShutdownGracePeriod):
                    // Grace period expired, exit
                    flushDirty()
                    return
                }
            }
        }
    // ... other cases
    }
}
```

---

## BUG #6: PARTIAL RECORD ON DISK FULL

### The Issue

```go
func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
    var hdr [8]byte
    binary.LittleEndian.PutUint32(hdr[0:4], walRecordMagic)
    binary.LittleEndian.PutUint32(hdr[4:8], uint32(payloadSize))
    if _, err := w.Write(hdr[:]); err != nil {
        return err  // <-- Partial data in buffer!
    }
    // ... more writes ...
}
```

If any write fails, previous writes sit in bufio.Writer. Next call appends, corrupting the file.

### Fix Strategy: Two-Phase Write

```go
func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
    // Phase 1: Prepare all data in a buffer
    var buf bytes.Buffer

    // Write to temporary buffer
    payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
    for _, s := range msg.Selector {
        payloadSize += 1 + len(s)
    }

    // Write magic + length
    var hdr [8]byte
    binary.LittleEndian.PutUint32(hdr[0:4], walRecordMagic)
    binary.LittleEndian.PutUint32(hdr[4:8], uint32(payloadSize))
    buf.Write(hdr[:])

    // ... prepare payload in buf ...

    // Phase 2: Compute CRC
    crc := crc32.NewIEEE()
    payload := buf.Bytes()[8:]  // Skip magic + length
    crc.Write(payload)

    // Phase 3: Atomic write (buffer is complete, write as one unit)
    if _, err := w.Write(buf.Bytes()); err != nil {
        return err
    }

    var crcBuf [4]byte
    binary.LittleEndian.PutUint32(crcBuf[:], crc.Sum32())
    if _, err := w.Write(crcBuf[:]); err != nil {
        return err
    }

    return nil
}
```

---

## BUG #8: NON-ATOMIC SNAPSHOT + WAL ROTATION

### The Issue

```
T1: Snapshot writes data up to timestamp T0
T2: Worker deletes current.wal
T3: Metric arrives with timestamp T0+100ms
T4: Metric written to current.wal
T5: Process crashes
T6: On restart: Snapshot from T0 + empty WAL
T7: Data from T0+100ms is lost!
```

### Fix Strategy: Atomic Rotation

**Option A: Rename + Replace**

```go
// In ToCheckpointWAL worker (line 697):
func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
    // ... write snapshot ...

    // Atomic rotation: rename current.wal to timestamp.wal.old
    walPath := path.Join(dir, "current.wal")
    oldPath := path.Join(dir, fmt.Sprintf("%d.wal.old", from))

    if err := os.Rename(walPath, oldPath); err != nil && !os.IsNotExist(err) {
        return fmt.Errorf("rename WAL: %w", err)
    }

    // old file will be deleted by cleanup job
    // or: delete it here if we're sure no reads are in progress
    os.Remove(oldPath)

    return nil
}
```

**Option B: Timestamp Marker**

```go
// Write checkpoint metadata with timestamp
type CheckpointMetadata struct {
    SnapshotTS int64  // Snapshot created up to this time
    CreatedAt  time.Time
}

// Store in .json or separate metadata file

// On load: Only replay WAL entries > SnapshotTS
```

**Option C: WAL Truncation Marker**

```go
// In current.wal, write a special marker record:
const walTruncateMarker = uint32(0xCC1DA7FF)

func writeWALTruncateMarker(w *bufio.Writer, ts int64) error {
    var hdr [12]byte
    binary.LittleEndian.PutUint32(hdr[0:4], walTruncateMarker)
    binary.LittleEndian.PutUint32(hdr[4:8], 8)
    binary.LittleEndian.PutUint64(hdr[8:16], uint64(ts))

    _, err := w.Write(hdr[:])
    return err
}

// On load: Stop replaying at truncate marker
```

---

## BUG #9: INITIALIZATION RACE

Same as BUG #2 analysis. Initialize walNumShards synchronously before NATS subscription.

---

## TESTING CHECKLIST

```go
// Test cases needed:
func TestShutdownWithInFlightMetrics(t *testing.T) {
    // Send metric, immediately shutdown
    // Verify no panic, metric persisted or acknowledged dropped
}

func TestRotationDeadlock(t *testing.T) {
    // Trigger checkpoint rotation during heavy write
    // Verify shutdown completes within timeout
}

func TestDiskFullRecovery(t *testing.T) {
    // Simulate disk full during WAL write
    // Verify WAL is not corrupted on restart
}

func TestNATSMetricsWithWALDisabled(t *testing.T) {
    // Run with WAL disabled
    // Verify no crashes
}

func TestBackpressure(t *testing.T) {
    // Send metrics faster than WAL can write
    // Verify either blocked or acknowledged dropped (not silent)
}

func TestSnapshotWALConsistency(t *testing.T) {
    // Write snapshot, then WAL, crash during gap
    // Verify data completeness on restart
}
```


Now let me verify the agent's claims against the actual code, especially the critical ones.


   125→// speed, amortizing syscall overhead across many writes.
   126→const walFlushInterval = 5 * time.Second
   127→
   128→// walShardIndex computes which shard a message belongs to based on cluster+node.
   129→// Uses FNV-1a hash for fast, well-distributed mapping.
   130→func walShardIndex(cluster, node string) int {
   131→	h := fnv.New32a()
   132→	h.Write([]byte(cluster))
   133→	h.Write([]byte{0})
   134→	h.Write([]byte(node))
   135→	return int(h.Sum32()) % walNumShards
   136→}
   137→
   138→// SendWALMessage routes a WAL message to the appropriate shard channel.
   139→// Returns false if the channel is full (message dropped).
   140→func SendWALMessage(msg *WALMessage) bool {
   141→	if walShardChs == nil {
   142→		return false
   143→	}
   144→	shard := walShardIndex(msg.Cluster, msg.Node)
   145→	select {
   146→	case walShardChs[shard] <- msg:
   147→		return true
   148→	default:
   149→		return false
   150→	}
   151→}
   152→
   153→// WALStaging starts N sharded background goroutines that receive WALMessage items
   154→// and append binary WAL records to per-host current.wal files.
   155→// Each shard owns a subset of hosts (determined by hash of cluster+node),
   156→// parallelizing serialization and I/O while keeping per-host writes serial.
   157→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   158→	if Keys.Checkpoints.FileFormat == "json" {
   159→		return

// speed, amortizing syscall overhead across many writes.
const walFlushInterval = 5 * time.Second

// walShardIndex computes which shard a message belongs to based on cluster+node.
// Uses FNV-1a hash for fast, well-distributed mapping.
func walShardIndex(cluster, node string) int {
	h := fnv.New32a()
	h.Write([]byte(cluster))
	h.Write([]byte{0})
	h.Write([]byte(node))
	return int(h.Sum32()) % walNumShards
}

// SendWALMessage routes a WAL message to the appropriate shard channel.
// Returns false if the channel is full (message dropped).
func SendWALMessage(msg *WALMessage) bool {
	if walShardChs == nil {
		return false
	}
	shard := walShardIndex(msg.Cluster, msg.Node)
	select {
	case walShardChs[shard] <- msg:
		return true
	default:
		return false
	}
}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.
// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return


    40→// callback decodes inline (no channel overhead, lower latency).
    41→//
    42→// The function blocks until ctx is cancelled and all worker goroutines have
    43→// finished. It returns nil when the NATS client is not configured; callers
    44→// should treat that as a no-op rather than an error.
    45→func ReceiveNats(ms *MemoryStore,
    46→	workers int,
    47→	ctx context.Context,
    48→) error {
    49→	nc := nats.GetClient()
    50→
    51→	if nc == nil {
    52→		cclog.Warn("NATS client not initialized")
    53→		return nil
    54→	}
    55→
    56→	var wg sync.WaitGroup
    57→	msgs := make(chan []byte, max(workers*256, 8192))
    58→
    59→	for _, sc := range *Keys.Subscriptions {
    60→		clusterTag := sc.ClusterTag
    61→		if workers > 1 {
    62→			wg.Add(workers)
    63→
    64→			for range workers {
    65→				go func() {
    66→					defer wg.Done()
    67→					for m := range msgs {
    68→						dec := lineprotocol.NewDecoderWithBytes(m)
    69→						if err := DecodeLine(dec, ms, clusterTag); err != nil {
    70→							cclog.Errorf("error: %s", err.Error())
    71→						}
    72→					}
    73→				}()
    74→			}
    75→
    76→			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
    77→				select {
    78→				case msgs <- data:
    79→				case <-ctx.Done():
    80→				}
    81→			})
    82→		} else {
    83→			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
    84→				dec := lineprotocol.NewDecoderWithBytes(data)
    85→				if err := DecodeLine(dec, ms, clusterTag); err != nil {
    86→					cclog.Errorf("error: %s", err.Error())
    87→				}
    88→			})
    89→		}
    90→		cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo)
    91→	}
    92→
    93→	go func() {
    94→		<-ctx.Done()
    95→		close(msgs)
    96→	}()
    97→
    98→	wg.Wait()
    99→
   100→	return nil
   101→}
   102→
   103→// reorder prepends prefix to buf in-place when buf has enough spare capacity,
   104→// avoiding an allocation. Falls back to a regular append otherwise.
   105→//
   106→// It is used to assemble the "type<type-id>" and "subtype<stype-id>" selector
   107→// strings when the type tag arrives before the type-id tag in the line, so the
   108→// two byte slices need to be concatenated in tag-declaration order regardless
   109→// of wire order.
   110→func reorder(buf, prefix []byte) []byte {
   111→	n := len(prefix)
   112→	m := len(buf)
   113→	if cap(buf) < m+n {
   114→		return append(prefix[:n:n], buf...)
   115→	} else {
   116→		buf = buf[:n+m]
   117→		for i := m - 1; i >= 0; i-- {
   118→			buf[i+n] = buf[i]
   119→		}

// callback decodes inline (no channel overhead, lower latency).
//
// The function blocks until ctx is cancelled and all worker goroutines have
// finished. It returns nil when the NATS client is not configured; callers
// should treat that as a no-op rather than an error.
func ReceiveNats(ms *MemoryStore,
	workers int,
	ctx context.Context,
) error {
	nc := nats.GetClient()

	if nc == nil {
		cclog.Warn("NATS client not initialized")
		return nil
	}

	var wg sync.WaitGroup
	msgs := make(chan []byte, max(workers*256, 8192))

	for _, sc := range *Keys.Subscriptions {
		clusterTag := sc.ClusterTag
		if workers > 1 {
			wg.Add(workers)

			for range workers {
				go func() {
					defer wg.Done()
					for m := range msgs {
						dec := lineprotocol.NewDecoderWithBytes(m)
						if err := DecodeLine(dec, ms, clusterTag); err != nil {
							cclog.Errorf("error: %s", err.Error())
						}
					}
				}()
			}

			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
				select {
				case msgs <- data:
				case <-ctx.Done():
				}
			})
		} else {
			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
				dec := lineprotocol.NewDecoderWithBytes(data)
				if err := DecodeLine(dec, ms, clusterTag); err != nil {
					cclog.Errorf("error: %s", err.Error())
				}
			})
		}
		cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo)
	}

	go func() {
		<-ctx.Done()
		close(msgs)
	}()

	wg.Wait()

	return nil
}

// reorder prepends prefix to buf in-place when buf has enough spare capacity,
// avoiding an allocation. Falls back to a regular append otherwise.
//
// It is used to assemble the "type<type-id>" and "subtype<stype-id>" selector
// strings when the type tag arrives before the type-id tag in the line, so the
// two byte slices need to be concatenated in tag-declaration order regardless
// of wire order.
func reorder(buf, prefix []byte) []byte {
	n := len(prefix)
	m := len(buf)
	if cap(buf) < m+n {
		return append(prefix[:n:n], buf...)
	} else {
		buf = buf[:n+m]
		for i := m - 1; i >= 0; i-- {
			buf[i+n] = buf[i]
		}


   340→				if t, err = dec.Time(lineprotocol.Microsecond, t); err != nil {
   341→					t = time.Now()
   342→					if t, err = dec.Time(lineprotocol.Nanosecond, t); err != nil {
   343→						return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
   344→					}
   345→				}
   346→			}
   347→		}
   348→
   349→		if err != nil {
   350→			return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
   351→		}
   352→
   353→		time := t.Unix()
   354→
   355→		if Keys.Checkpoints.FileFormat == "wal" {
   356→			msg := &WALMessage{
   357→				MetricName: string(st.metricBuf),
   358→				Cluster:    cluster,
   359→				Node:       host,
   360→				Selector:   append([]string{}, st.selector...),
   361→				Value:      metric.Value,
   362→				Timestamp:  time,
   363→			}
   364→			if !SendWALMessage(msg) {
   365→				// WAL shard channel full — metric is written to memory store but not WAL.
   366→				// Next binary snapshot will capture it.
   367→				if dropped := walDropped.Add(1); dropped%10000 == 1 {
   368→					cclog.Warnf("[METRICSTORE]> WAL shard channel full, dropped %d messages (data safe in memory, next snapshot will capture)", dropped)
   369→				}
   370→			}
   371→		}
   372→
   373→		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
   374→			cclog.Warnf("write error for host %s metric %s at ts %d: %s", host, string(st.metricBuf), time, err.Error())
   375→			continue
   376→		}
   377→	}
   378→	return nil
   379→}

				if t, err = dec.Time(lineprotocol.Microsecond, t); err != nil {
					t = time.Now()
					if t, err = dec.Time(lineprotocol.Nanosecond, t); err != nil {
						return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
					}
				}
			}
		}

		if err != nil {
			return fmt.Errorf("host %s: timestamp : %#v with error : %#v", host, t, err.Error())
		}

		time := t.Unix()

		if Keys.Checkpoints.FileFormat == "wal" {
			msg := &WALMessage{
				MetricName: string(st.metricBuf),
				Cluster:    cluster,
				Node:       host,
				Selector:   append([]string{}, st.selector...),
				Value:      metric.Value,
				Timestamp:  time,
			}
			if !SendWALMessage(msg) {
				// WAL shard channel full — metric is written to memory store but not WAL.
				// Next binary snapshot will capture it.
				if dropped := walDropped.Add(1); dropped%10000 == 1 {
					cclog.Warnf("[METRICSTORE]> WAL shard channel full, dropped %d messages (data safe in memory, next snapshot will capture)", dropped)
				}
			}
		}

		if err := ms.WriteToLevel(lvl, st.selector, time, []Metric{metric}); err != nil {
			cclog.Warnf("write error for host %s metric %s at ts %d: %s", host, string(st.metricBuf), time, err.Error())
			continue
		}
	}
	return nil
}


   145→	ms := GetMemoryStore()
   146→
   147→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   148→	if err != nil {
   149→		cclog.Fatal(err)
   150→	}
   151→
   152→	restoreFrom := startupTime.Add(-d)
   153→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   154→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   155→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   156→	if err != nil {
   157→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   158→	} else {
   159→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   160→	}
   161→
   162→	// Try to use less memory by forcing a GC run here and then
   163→	// lowering the target percentage. The default of 100 means
   164→	// that only once the ratio of new allocations execeds the
   165→	// previously active heap, a GC is triggered.
   166→	// Forcing a GC here will set the "previously active heap"
   167→	// to a minumum.
   168→	// runtime.GC()
   169→
   170→	ctx, shutdown := context.WithCancel(context.Background())
   171→
   172→	Retention(wg, ctx)
   173→	Checkpointing(wg, ctx)
   174→	CleanUp(wg, ctx)
   175→	WALStaging(wg, ctx)
   176→	MemoryUsageTracker(wg, ctx)
   177→
   178→	// Note: Signal handling has been removed from this function.
   179→	// The caller is responsible for handling shutdown signals and calling
   180→	// the shutdown() function when appropriate.
   181→	// Store the shutdown function for later use by Shutdown()
   182→	shutdownFuncMu.Lock()
   183→	shutdownFunc = shutdown
   184→	shutdownFuncMu.Unlock()
   185→
   186→	if Keys.Subscriptions != nil {
   187→		wg.Go(func() {
   188→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   189→				cclog.Fatal(err)
   190→			}
   191→		})
   192→	}
   193→}
   194→

	ms := GetMemoryStore()

	d, err := time.ParseDuration(Keys.RetentionInMemory)
	if err != nil {
		cclog.Fatal(err)
	}

	restoreFrom := startupTime.Add(-d)
	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
	if err != nil {
		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
	} else {
		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
	}

	// Try to use less memory by forcing a GC run here and then
	// lowering the target percentage. The default of 100 means
	// that only once the ratio of new allocations execeds the
	// previously active heap, a GC is triggered.
	// Forcing a GC here will set the "previously active heap"
	// to a minumum.
	// runtime.GC()

	ctx, shutdown := context.WithCancel(context.Background())

	Retention(wg, ctx)
	Checkpointing(wg, ctx)
	CleanUp(wg, ctx)
	WALStaging(wg, ctx)
	MemoryUsageTracker(wg, ctx)

	// Note: Signal handling has been removed from this function.
	// The caller is responsible for handling shutdown signals and calling
	// the shutdown() function when appropriate.
	// Store the shutdown function for later use by Shutdown()
	shutdownFuncMu.Lock()
	shutdownFunc = shutdown
	shutdownFuncMu.Unlock()

	if Keys.Subscriptions != nil {
		wg.Go(func() {
			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
				cclog.Fatal(err)
			}
		})
	}
}


Good. Now I have verified the claims. Let me assess which are real and which are not:


   157→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   158→	if Keys.Checkpoints.FileFormat == "json" {
   159→		return
   160→	}
   161→
   162→	walNumShards = max(Keys.NumWorkers, 1)
   163→	chBufSize := max(65536/walNumShards, 1024)
   164→
   165→	walShardChs = make([]chan *WALMessage, walNumShards)
   166→	walShardRotateChs = make([]chan walRotateReq, walNumShards)
   167→
   168→	for i := range walNumShards {
   169→		walShardChs[i] = make(chan *WALMessage, chBufSize)
   170→		walShardRotateChs[i] = make(chan walRotateReq, 64)
   171→	}

func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return
	}

	walNumShards = max(Keys.NumWorkers, 1)
	chBufSize := max(65536/walNumShards, 1024)

	walShardChs = make([]chan *WALMessage, walNumShards)
	walShardRotateChs = make([]chan walRotateReq, walNumShards)

	for i := range walNumShards {
		walShardChs[i] = make(chan *WALMessage, chBufSize)
		walShardRotateChs[i] = make(chan walRotateReq, 64)
	}


Let me verify a few more specific code paths the agent flagged.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements ingestion of InfluxDB line-protocol metric data received
     7→// over NATS. Each line encodes one metric sample with the following structure:
     8→//
     9→//	<measurement>[,cluster=<c>][,hostname=<h>][,type=<t>][,type-id=<id>][,subtype=<s>][,stype-id=<id>] value=<v> [<timestamp>]
    10→//
    11→// The measurement name identifies the metric (e.g. "cpu_load"). Tags provide
    12→// routing information (cluster, host) and optional sub-device selectors (type,
    13→// subtype). Only one field is expected per line: "value".
    14→//
    15→// After decoding, each sample is:
    16→//  1. Written to the in-memory store via ms.WriteToLevel.
    17→//  2. If the checkpoint format is "wal", also forwarded to the WAL staging
    18→//     goroutine via the WALMessages channel for durable write-ahead logging.
    19→package metricstore
    20→
    21→import (
    22→	"bytes"
    23→	"context"
    24→	"fmt"
    25→	"sync"
    26→	"sync/atomic"
    27→	"time"
    28→
    29→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    30→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    31→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    32→	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
    33→)
    34→
    35→// ReceiveNats subscribes to all configured NATS subjects and feeds incoming
    36→// line-protocol messages into the MemoryStore.
    37→//
    38→// When workers > 1 a pool of goroutines drains a shared channel so that
    39→// multiple messages can be decoded in parallel. With workers == 1 the NATS
    40→// callback decodes inline (no channel overhead, lower latency).

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements ingestion of InfluxDB line-protocol metric data received
// over NATS. Each line encodes one metric sample with the following structure:
//
//	<measurement>[,cluster=<c>][,hostname=<h>][,type=<t>][,type-id=<id>][,subtype=<s>][,stype-id=<id>] value=<v> [<timestamp>]
//
// The measurement name identifies the metric (e.g. "cpu_load"). Tags provide
// routing information (cluster, host) and optional sub-device selectors (type,
// subtype). Only one field is expected per line: "value".
//
// After decoding, each sample is:
//  1. Written to the in-memory store via ms.WriteToLevel.
//  2. If the checkpoint format is "wal", also forwarded to the WAL staging
//     goroutine via the WALMessages channel for durable write-ahead logging.
package metricstore

import (
	"bytes"
	"context"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
)

// ReceiveNats subscribes to all configured NATS subjects and feeds incoming
// line-protocol messages into the MemoryStore.
//
// When workers > 1 a pool of goroutines drains a shared channel so that
// multiple messages can be decoded in parallel. With workers == 1 the NATS
// callback decodes inline (no channel overhead, lower latency).


    95→// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
    96→var walStagingWg sync.WaitGroup
    97→
    98→// WALMessage represents a single metric write to be appended to the WAL.
    99→// Cluster and Node are NOT stored in the WAL record (inferred from file path).
   100→type WALMessage struct {
   101→	MetricName string
   102→	Cluster    string
   103→	Node       string
   104→	Selector   []string
   105→	Value      schema.Float
   106→	Timestamp  int64
   107→}
   108→
   109→// walRotateReq requests WAL file rotation for a specific host directory.
   110→// The done channel is closed by the WAL goroutine when rotation is complete.
   111→type walRotateReq struct {
   112→	hostDir string
   113→	done    chan struct{}
   114→}
   115→
   116→// walFileState holds an open WAL file handle and buffered writer for one host directory.
   117→type walFileState struct {
   118→	f     *os.File
   119→	w     *bufio.Writer
   120→	dirty bool
   121→}
   122→
   123→// walFlushInterval controls how often dirty WAL files are flushed to disk.
   124→// Decoupling flushes from message processing lets the consumer run at memory
   125→// speed, amortizing syscall overhead across many writes.
   126→const walFlushInterval = 5 * time.Second
   127→
   128→// walShardIndex computes which shard a message belongs to based on cluster+node.
   129→// Uses FNV-1a hash for fast, well-distributed mapping.
   130→func walShardIndex(cluster, node string) int {
   131→	h := fnv.New32a()
   132→	h.Write([]byte(cluster))
   133→	h.Write([]byte{0})
   134→	h.Write([]byte(node))
   135→	return int(h.Sum32()) % walNumShards
   136→}
   137→
   138→// SendWALMessage routes a WAL message to the appropriate shard channel.
   139→// Returns false if the channel is full (message dropped).
   140→func SendWALMessage(msg *WALMessage) bool {
   141→	if walShardChs == nil {
   142→		return false
   143→	}
   144→	shard := walShardIndex(msg.Cluster, msg.Node)
   145→	select {
   146→	case walShardChs[shard] <- msg:
   147→		return true
   148→	default:
   149→		return false
   150→	}
   151→}
   152→
   153→// WALStaging starts N sharded background goroutines that receive WALMessage items
   154→// and append binary WAL records to per-host current.wal files.

// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
var walStagingWg sync.WaitGroup

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f     *os.File
	w     *bufio.Writer
	dirty bool
}

// walFlushInterval controls how often dirty WAL files are flushed to disk.
// Decoupling flushes from message processing lets the consumer run at memory
// speed, amortizing syscall overhead across many writes.
const walFlushInterval = 5 * time.Second

// walShardIndex computes which shard a message belongs to based on cluster+node.
// Uses FNV-1a hash for fast, well-distributed mapping.
func walShardIndex(cluster, node string) int {
	h := fnv.New32a()
	h.Write([]byte(cluster))
	h.Write([]byte{0})
	h.Write([]byte(node))
	return int(h.Sum32()) % walNumShards
}

// SendWALMessage routes a WAL message to the appropriate shard channel.
// Returns false if the channel is full (message dropped).
func SendWALMessage(msg *WALMessage) bool {
	if walShardChs == nil {
		return false
	}
	shard := walShardIndex(msg.Cluster, msg.Node)
	select {
	case walShardChs[shard] <- msg:
		return true
	default:
		return false
	}
}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.


Now I have the full picture. Here's the review with verified findings:

---

## In-Depth Review of the Sharded WAL Buffer Pipeline

I verified every claim from the exploration agent against the actual code. Here's what's real and what's not.

### CRITICAL: Send-on-closed-channel panic at shutdown (BUG #1)

**Real and exploitable.** This is the most dangerous issue.

`Shutdown()` (`metricstore.go:273`) does:
1. Cancel context (`shutdownFunc()`)
2. Immediately close `walShardChs[i]`

Meanwhile, `ReceiveNats` workers (`lineprotocol.go:64-73`) read from the `msgs` channel. When ctx is cancelled, a goroutine closes `msgs` — but workers that already dequeued a message are still calling `DecodeLine` → `SendWALMessage`. `SendWALMessage` does `case walShardChs[shard] <- msg:` — **sending on a closed channel panics in Go, even inside a `select` with `default`.**

The race window: between `shutdownFunc()` and `close(walShardChs[i])`, NATS workers can still be mid-decode. The `walShardChs == nil` guard at line 141 doesn't help — the channel isn't nil, it's closed.

**Fix**: Add an `atomic.Bool` shutdown flag checked in `SendWALMessage` before the send, set it before closing channels, and add a small grace period.

### MEDIUM: Periodic checkpoint ↔ shutdown race on rotation channels (BUG #3)

**Real but low likelihood.** If the `Checkpointing` goroutine's ticker fires at the same moment as `ctx.Done()`, Go's `select` picks randomly. If the ticker wins, the goroutine enters `ToCheckpointWAL` → `RotateWALFiles`. Meanwhile, WALStaging goroutines see `ctx.Done()` and exit. `RotateWALFiles` sends to `walShardRotateChs` (never closed), then blocks forever on `<-done` since nobody processes the request. The Checkpointing goroutine hangs, blocking the main WaitGroup.

**Fix**: Either add a timeout to `RotateWALFiles`, or check ctx before calling it in the Checkpointing loop.

### NOT REAL: Division by zero on startup (BUG #2)

**False alarm.** `WALStaging()` initializes `walNumShards` and creates channels **synchronously** at lines 162-170 before returning. `Init()` calls `WALStaging()` at line 175, which completes synchronously. Only then does `ReceiveNats` start (line 187, via `wg.Go` — a new goroutine). By the time any NATS message arrives, `walNumShards` is set.

### NOT REAL: Lost metrics at shutdown (BUG #5)

**False alarm.** The agent claimed skipping the WAL drain loses data. But `DecodeLine` (`lineprotocol.go:355-376`) writes to memory first (`WriteToLevel`) regardless of WAL success. `ToCheckpointWAL` reads from the **in-memory state**, not from WAL. The skipped WAL messages are truly redundant — the same data is already in memory and captured by the final snapshot.

### DESIGN TRADE-OFF: WAL drops under backpressure (BUG #4)

**By design, not a bug.** The code comment at line 366 correctly states: "data safe in memory, next snapshot will capture." The WAL is a crash-recovery mechanism. Dropped WAL messages only matter if the process crashes before the next snapshot. The logging at every 10k drops is adequate. The real question is whether the channel size (65536/shards) is appropriately sized for your workload.

### MEDIUM: Partial WAL record on write error (BUG #6)

**Real.** `writeWALRecordDirect` writes multiple chunks to a `bufio.Writer`. If a write fails mid-record (e.g., disk full), partial data remains in the buffer. The next call appends, creating corruption. The CRC check during replay will detect corrupted records, so data integrity is preserved at the read side — but the WAL file becomes partially unreadable.

**Fix**: Pre-compute the full record into a byte slice, then write atomically.

### LOW: Non-atomic snapshot + WAL delete (BUG #8)

**Partially mitigated by our changes.** After our fix, WAL staging goroutines have exited before `ToCheckpointWAL` runs, so no new WAL records are being written. The delete-after-snapshot-write in the worker is safe. The remaining risk is during **periodic** checkpoints (not shutdown), where WAL staging is still active. A crash between snapshot write and WAL delete could lose metrics in that window. This is an inherent trade-off of the two-file design.

### Summary of actionable items

| Priority | Issue | Fix complexity |
|----------|-------|----------------|
| **CRITICAL** | Send-on-closed-channel panic (#1) | Add `atomic.Bool` flag + grace period |
| **MEDIUM** | Periodic checkpoint rotation deadlock (#3) | Add timeout to `RotateWALFiles` |
| **MEDIUM** | Partial WAL record on disk full (#6) | Buffer full record before write |
| **LOW** | Non-atomic periodic snapshot+WAL (#8) | Accept or add WAL truncation marker |

Want me to fix the critical send-on-closed-channel issue?


Fix the critical and medium issues


TaskCreate

Fix send-on-closed-channel panic at shutdown

Add atomic.Bool shutdown flag to SendWALMessage, set before closing channels


Task #6 created successfully: Fix send-on-closed-channel panic at shutdown


Fix periodic checkpoint rotation deadlock

Add timeout to RotateWALFiles to prevent hanging if WAL staging goroutines exit


Task #7 created successfully: Fix periodic checkpoint rotation deadlock


Fix partial WAL record on write error

Pre-compute full WAL record into buffer before writing atomically


Task #8 created successfully: Fix partial WAL record on write error

Fix partial WAL record on write error


Build and test all fixes

go build, go vet, go test


Task #9 created successfully: Build and test all fixes

Build and test all fixes


Updated task #6 status


    85→// walShardChs holds per-shard channels for WAL messages.
    86→// Initialized by WALStaging; nil when WAL is not active.
    87→var walShardChs []chan *WALMessage
    88→
    89→// walShardRotateChs holds per-shard channels for WAL rotation requests.
    90→var walShardRotateChs []chan walRotateReq
    91→
    92→// walNumShards stores the number of shards (set during WALStaging init).
    93→var walNumShards int
    94→
    95→// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
    96→var walStagingWg sync.WaitGroup
    97→
    98→// WALMessage represents a single metric write to be appended to the WAL.
    99→// Cluster and Node are NOT stored in the WAL record (inferred from file path).
   100→type WALMessage struct {
   101→	MetricName string
   102→	Cluster    string
   103→	Node       string
   104→	Selector   []string
   105→	Value      schema.Float
   106→	Timestamp  int64
   107→}
   108→
   109→// walRotateReq requests WAL file rotation for a specific host directory.
   110→// The done channel is closed by the WAL goroutine when rotation is complete.
   111→type walRotateReq struct {
   112→	hostDir string
   113→	done    chan struct{}
   114→}
   115→
   116→// walFileState holds an open WAL file handle and buffered writer for one host directory.
   117→type walFileState struct {
   118→	f     *os.File
   119→	w     *bufio.Writer
   120→	dirty bool
   121→}
   122→
   123→// walFlushInterval controls how often dirty WAL files are flushed to disk.
   124→// Decoupling flushes from message processing lets the consumer run at memory
   125→// speed, amortizing syscall overhead across many writes.
   126→const walFlushInterval = 5 * time.Second
   127→
   128→// walShardIndex computes which shard a message belongs to based on cluster+node.
   129→// Uses FNV-1a hash for fast, well-distributed mapping.
   130→func walShardIndex(cluster, node string) int {
   131→	h := fnv.New32a()
   132→	h.Write([]byte(cluster))
   133→	h.Write([]byte{0})
   134→	h.Write([]byte(node))
   135→	return int(h.Sum32()) % walNumShards
   136→}
   137→
   138→// SendWALMessage routes a WAL message to the appropriate shard channel.
   139→// Returns false if the channel is full (message dropped).
   140→func SendWALMessage(msg *WALMessage) bool {
   141→	if walShardChs == nil {
   142→		return false
   143→	}
   144→	shard := walShardIndex(msg.Cluster, msg.Node)
   145→	select {
   146→	case walShardChs[shard] <- msg:
   147→		return true
   148→	default:
   149→		return false
   150→	}
   151→}
   152→
   153→// WALStaging starts N sharded background goroutines that receive WALMessage items
   154→// and append binary WAL records to per-host current.wal files.

// walShardChs holds per-shard channels for WAL messages.
// Initialized by WALStaging; nil when WAL is not active.
var walShardChs []chan *WALMessage

// walShardRotateChs holds per-shard channels for WAL rotation requests.
var walShardRotateChs []chan walRotateReq

// walNumShards stores the number of shards (set during WALStaging init).
var walNumShards int

// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
var walStagingWg sync.WaitGroup

// WALMessage represents a single metric write to be appended to the WAL.
// Cluster and Node are NOT stored in the WAL record (inferred from file path).
type WALMessage struct {
	MetricName string
	Cluster    string
	Node       string
	Selector   []string
	Value      schema.Float
	Timestamp  int64
}

// walRotateReq requests WAL file rotation for a specific host directory.
// The done channel is closed by the WAL goroutine when rotation is complete.
type walRotateReq struct {
	hostDir string
	done    chan struct{}
}

// walFileState holds an open WAL file handle and buffered writer for one host directory.
type walFileState struct {
	f     *os.File
	w     *bufio.Writer
	dirty bool
}

// walFlushInterval controls how often dirty WAL files are flushed to disk.
// Decoupling flushes from message processing lets the consumer run at memory
// speed, amortizing syscall overhead across many writes.
const walFlushInterval = 5 * time.Second

// walShardIndex computes which shard a message belongs to based on cluster+node.
// Uses FNV-1a hash for fast, well-distributed mapping.
func walShardIndex(cluster, node string) int {
	h := fnv.New32a()
	h.Write([]byte(cluster))
	h.Write([]byte{0})
	h.Write([]byte(node))
	return int(h.Sum32()) % walNumShards
}

// SendWALMessage routes a WAL message to the appropriate shard channel.
// Returns false if the channel is full (message dropped).
func SendWALMessage(msg *WALMessage) bool {
	if walShardChs == nil {
		return false
	}
	shard := walShardIndex(msg.Cluster, msg.Node)
	select {
	case walShardChs[shard] <- msg:
		return true
	default:
		return false
	}
}

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.


Add the `walShuttingDown` atomic flag and update `SendWALMessage`:


Edit

// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
var walStagingWg sync.WaitGroup

// walStagingWg tracks WALStaging goroutine exits for shutdown synchronization.
var walStagingWg sync.WaitGroup

// walShuttingDown is set before closing shard channels to prevent
// SendWALMessage from sending on a closed channel (which panics in Go).
var walShuttingDown atomic.Bool