   300→			RotateWALFilesAfterShutdown(hostDirs)
   301→		}
   302→	} else {
   303→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   304→	}
   305→
   306→	if err != nil {
   307→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
   308→	}
   309→	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
   310→}
   311→
   312→// Retention starts a background goroutine that periodically frees old metric data.
   313→//
   314→// This worker runs at half the retention interval and calls Free() to remove buffers
   315→// older than the configured retention time. It respects the NodeProvider to preserve
   316→// data for nodes with active jobs.
   317→//
   318→// Parameters:
   319→//   - wg: WaitGroup to signal completion when context is cancelled
   320→//   - ctx: Context for cancellation signal
   321→//
   322→// The goroutine exits when ctx is cancelled.
   323→func Retention(wg *sync.WaitGroup, ctx context.Context) {
   324→	ms := GetMemoryStore()
   325→
   326→	wg.Go(func() {
   327→		d, err := time.ParseDuration(Keys.RetentionInMemory)
   328→		if err != nil {
   329→			cclog.Fatal(err)
   330→		}
   331→		if d <= 0 {
   332→			return
   333→		}
   334→
   335→		tickInterval := d / 2
   336→		if tickInterval <= 0 {
   337→			return
   338→		}
   339→		ticker := time.NewTicker(tickInterval)
   340→		defer ticker.Stop()
   341→
   342→		for {
   343→			select {
   344→			case <-ctx.Done():
   345→				return
   346→			case <-ticker.C:
   347→				state.mu.Lock()
   348→
   349→				t := time.Now().Add(-d)
   350→
   351→				state.lastRetentionTime = t.Unix()
   352→
   353→				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
   354→
   355→				freed, err := Free(ms, t)
   356→				if err != nil {
   357→					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
   358→				} else {
   359→					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
   360→				}
   361→
   362→				state.mu.Unlock()
   363→
   364→				// Clean up the buffer pool
   365→				bufferPool.Clean(state.lastRetentionTime)
   366→			}
   367→		}
   368→	})
   369→}
   370→
   371→// MemoryUsageTracker starts a background goroutine that monitors memory usage.
   372→//
   373→// This worker checks actual process memory usage (via runtime.MemStats) periodically
   374→// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
   375→// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
   376→// performance issues.
   377→//
   378→// The tracker logs both actual memory usage (heap allocated) and metric data size for
   379→// visibility into memory overhead from Go runtime structures and allocations.
   380→//
   381→// Parameters:
   382→//   - wg: WaitGroup to signal completion when context is cancelled
   383→//   - ctx: Context for cancellation signal
   384→//
   385→// The goroutine exits when ctx is cancelled.
   386→func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
   387→	ms := GetMemoryStore()
   388→
   389→	wg.Go(func() {
   390→		normalInterval := DefaultMemoryUsageTrackerInterval
   391→		fastInterval := 30 * time.Second
   392→
   393→		if normalInterval <= 0 {
   394→			return
   395→		}
   396→
   397→		ticker := time.NewTicker(normalInterval)
   398→		defer ticker.Stop()
   399→		currentInterval := normalInterval
   400→
   401→		for {
   402→			select {
   403→			case <-ctx.Done():
   404→				return
   405→			case <-ticker.C:
   406→				var mem runtime.MemStats
   407→				runtime.ReadMemStats(&mem)
   408→				actualMemoryGB := float64(mem.Alloc) / 1e9
   409→				metricDataGB := ms.SizeInGB()
   410→				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
   411→
   412→				freedExcluded := 0
   413→				freedEmergency := 0
   414→				var err error
   415→
   416→				state.mu.RLock()
   417→				lastRetention := state.lastRetentionTime
   418→				selectorsExcluded := state.selectorsExcluded
   419→				state.mu.RUnlock()
   420→
   421→				if lastRetention != 0 && selectorsExcluded {
   422→					freedExcluded, err = ms.Free(nil, lastRetention)
   423→					if err != nil {
   424→						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
   425→					}
   426→
   427→					if freedExcluded > 0 {
   428→						debug.FreeOSMemory()
   429→						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
   430→					}
   431→				}
   432→
   433→				runtime.ReadMemStats(&mem)
   434→				actualMemoryGB = float64(mem.Alloc) / 1e9
   435→
   436→				if actualMemoryGB > float64(Keys.MemoryCap) {
   437→					cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)
   438→
   439→					// Use progressive time-based Free with increasing threshold
   440→					// instead of ForceFree loop — fewer tree traversals, more effective
   441→					d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
   442→					if parseErr != nil {
   443→						cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
   444→					} else {
   445→						thresholds := []float64{0.75, 0.5, 0.25}
   446→						for _, fraction := range thresholds {
   447→							threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
   448→							freed, freeErr := ms.Free(nil, threshold.Unix())
   449→							if freeErr != nil {
   450→								cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
   451→							}
   452→							freedEmergency += freed
   453→
   454→							bufferPool.Clear()
   455→							runtime.GC()
   456→							runtime.ReadMemStats(&mem)
   457→							actualMemoryGB = float64(mem.Alloc) / 1e9
   458→
   459→							if actualMemoryGB < float64(Keys.MemoryCap) {
   460→								break
   461→							}
   462→						}
   463→					}
   464→
   465→					bufferPool.Clear()
   466→					debug.FreeOSMemory()
   467→
   468→					runtime.ReadMemStats(&mem)
   469→					actualMemoryGB = float64(mem.Alloc) / 1e9
   470→
   471→					if actualMemoryGB >= float64(Keys.MemoryCap) {
   472→						cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
   473→					} else {
   474→						cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
   475→					}
   476→				}
   477→
   478→				// Adaptive ticker: check more frequently when memory is high
   479→				memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
   480→				if memoryRatio > 0.8 && currentInterval != fastInterval {
   481→					ticker.Reset(fastInterval)
   482→					currentInterval = fastInterval
   483→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
   484→				} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
   485→					ticker.Reset(normalInterval)
   486→					currentInterval = normalInterval
   487→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
   488→				}
   489→			}
   490→		}
   491→	})
   492→}
   493→
   494→// Free removes metric data older than the given time while preserving data for active nodes.
   495→//
   496→// This function implements intelligent retention by consulting the NodeProvider (if configured)
   497→// to determine which nodes are currently in use by running jobs. Data for these nodes is
   498→// preserved even if older than the retention time.
   499→//
   500→// Parameters:
   501→//   - ms: The MemoryStore instance
   502→//   - t: Time threshold - buffers with data older than this will be freed
   503→//
   504→// Returns:
   505→//   - Number of buffers freed
   506→//   - Error if NodeProvider query fails
   507→//
   508→// Behavior:
   509→//   - If no NodeProvider is set: frees all buffers older than t
   510→//   - If NodeProvider returns empty map: frees all buffers older than t
   511→//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
   512→func Free(ms *MemoryStore, t time.Time) (int, error) {
   513→	// If no NodeProvider is configured, free all buffers older than t
   514→	if ms.nodeProvider == nil {
   515→		return ms.Free(nil, t.Unix())
   516→	}
   517→
   518→	excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
   519→	if err != nil {
   520→		return 0, err
   521→	}
   522→
   523→	switch lenMap := len(excludeSelectors); lenMap {
   524→
   525→	// If the length of the map returned by GetUsedNodes() is 0,
   526→	// then use default Free method with nil selector
   527→	case 0:
   528→		state.selectorsExcluded = false
   529→		return ms.Free(nil, t.Unix())
   530→
   531→	// Else formulate selectors, exclude those from the map
   532→	// and free the rest of the selectors
   533→	default:
   534→		state.selectorsExcluded = true
   535→		selectors := GetSelectors(ms, excludeSelectors)
   536→		return FreeSelected(ms, selectors, t)
   537→	}
   538→}
   539→
   540→// FreeSelected frees buffers for specific selectors while preserving others.
   541→//
   542→// This function is used when we want to retain some specific nodes beyond the retention time.
   543→// It iterates through the provided selectors and frees their associated buffers.
   544→//
   545→// Parameters:
   546→//   - ms: The MemoryStore instance
   547→//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
   548→//   - t: Time threshold for freeing buffers
   549→//
   550→// Returns the total number of buffers freed and any error encountered.
   551→func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
   552→	freed := 0
   553→
   554→	for _, selector := range selectors {
   555→
   556→		freedBuffers, err := ms.Free(selector, t.Unix())
   557→		if err != nil {
   558→			cclog.Errorf("error while freeing selected buffers: %#v", err)
   559→		}
   560→		freed += freedBuffers
   561→
   562→	}
   563→
   564→	return freed, nil
   565→}
   566→
   567→// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
   568→//
   569→// This function generates a list of selectors whose buffers should be freed by excluding
   570→// selectors that correspond to nodes currently in use by running jobs.
   571→//
   572→// Parameters:
   573→//   - ms: The MemoryStore instance
   574→//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
   575→//
   576→// Returns a list of selectors ([]string paths) that can be safely freed.
   577→//
   578→// Example:
   579→//
   580→//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
   581→//	and excludeSelectors contains {"emmy": ["node001"]},
   582→//	then only [["emmy", "node002"]] is returned.
   583→func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
   584→	allSelectors := ms.GetPaths(2)
   585→
   586→	filteredSelectors := make([][]string, 0, len(allSelectors))
   587→
   588→	for _, path := range allSelectors {
   589→		if len(path) < 2 {
   590→			continue
   591→		}
   592→
   593→		key := path[0]   // The "Key" (Level 1)
   594→		value := path[1] // The "Value" (Level 2)
   595→
   596→		exclude := false
   597→
   598→		// Check if the key exists in our exclusion map
   599→		if excludedValues, exists := excludeSelectors[key]; exists {
   600→			// The key exists, now check if the specific value is in the exclusion list
   601→			if slices.Contains(excludedValues, value) {
   602→				exclude = true
   603→			}
   604→		}
   605→
   606→		if !exclude {
   607→			filteredSelectors = append(filteredSelectors, path)
   608→		}
   609→	}
   610→
   611→	return filteredSelectors
   612→}
   613→
   614→// GetPaths returns a list of lists (paths) to the specified depth.
   615→func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
   616→	var results [][]string
   617→
   618→	// Start recursion. Initial path is empty.
   619→	// We treat Root as depth 0.
   620→	ms.root.collectPaths(0, targetDepth, []string{}, &results)
   621→
   622→	return results
   623→}
   624→
   625→// Write all values in `metrics` to the level specified by `selector` for time `ts`.
   626→// Look at `findLevelOrCreate` for how selectors work.
   627→func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
   628→	var ok bool
   629→	for i, metric := range metrics {
   630→		if metric.MetricConfig.Frequency == 0 {
   631→			metric.MetricConfig, ok = m.Metrics[metric.Name]
   632→			if !ok {
   633→				cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
   634→				metric.MetricConfig.Frequency = 0
   635→			}
   636→			metrics[i] = metric
   637→		}
   638→	}
   639→
   640→	return m.WriteToLevel(&m.root, selector, ts, metrics)
   641→}
   642→
   643→func (m *MemoryStore) GetLevel(selector []string) *Level {
   644→	return m.root.findLevelOrCreate(selector, len(m.Metrics))
   645→}
   646→
   647→// WriteToLevel assumes that `minfo` in `metrics` is filled in
   648→func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
   649→	l = l.findLevelOrCreate(selector, len(m.Metrics))
   650→	l.lock.Lock()
   651→	defer l.lock.Unlock()
   652→
   653→	for _, metric := range metrics {
   654→		if metric.MetricConfig.Frequency == 0 {
   655→			continue
   656→		}
   657→
   658→		b := l.metrics[metric.MetricConfig.offset]
   659→		if b == nil {
   660→			// First write to this metric and level
   661→			b = newBuffer(ts, metric.MetricConfig.Frequency)
   662→			l.metrics[metric.MetricConfig.offset] = b
   663→		}
   664→
   665→		nb, err := b.write(ts, metric.Value)
   666→		if err != nil {
   667→			return err
   668→		}
   669→
   670→		// Last write created a new buffer...
   671→		if b != nb {
   672→			l.metrics[metric.MetricConfig.offset] = nb
   673→		}
   674→	}
   675→	return nil
   676→}
   677→
   678→// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
   679→// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
   680→// The second and third return value are the actual from/to for the data. Those can be different from
   681→// the range asked for if no data was available.
   682→func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
   683→	if from > to {
   684→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
   685→	}
   686→
   687→	minfo, ok := m.Metrics[metric]
   688→	if !ok {
   689→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
   690→	}
   691→
   692→	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
   693→
   694→	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
   695→		cdata, cfrom, cto, err := b.read(from, to, data)
   696→		if err != nil {
   697→			return err
   698→		}
   699→
   700→		if n == 0 {
   701→			from, to = cfrom, cto
   702→		} else if from != cfrom || to != cto || len(data) != len(cdata) {
   703→			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
   704→			if missingfront != 0 {
   705→				return ErrDataDoesNotAlign
   706→			}
   707→
   708→			newlen := len(cdata) - missingback
   709→			if newlen < 1 {
   710→				return ErrDataDoesNotAlign
   711→			}
   712→			cdata = cdata[0:newlen]
   713→			if len(cdata) != len(data) {
   714→				return ErrDataDoesNotAlign
   715→			}
   716→
   717→			from, to = cfrom, cto
   718→		}
   719→
   720→		data = cdata
   721→		n += 1
   722→		return nil
   723→	})
   724→
   725→	if err != nil {
   726→		return nil, 0, 0, 0, err
   727→	} else if n == 0 {
   728→		return nil, 0, 0, 0, ErrNoHostOrMetric
   729→	} else if n > 1 {
   730→		if minfo.Aggregation == AvgAggregation {
   731→			normalize := 1. / schema.Float(n)
   732→			for i := 0; i < len(data); i++ {
   733→				data[i] *= normalize
   734→			}
   735→		} else if minfo.Aggregation != SumAggregation {
   736→			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
   737→		}
   738→	}
   739→
   740→	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
   741→	if err != nil {
   742→		return nil, 0, 0, 0, err
   743→	}
   744→
   745→	return data, from, to, resolution, nil
   746→}
   747→
   748→// Free releases all buffers for the selected level and all its children that
   749→// contain only values older than `t`.
   750→func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
   751→	return m.GetLevel(selector).free(t)
   752→}
   753→
   754→// ForceFree unconditionally removes the oldest buffer from each metric chain.
   755→func (m *MemoryStore) ForceFree() (int, error) {
   756→	return m.GetLevel(nil).forceFree()
   757→}
   758→
   759→func (m *MemoryStore) FreeAll() error {
   760→	m.root.lock.Lock()
   761→	defer m.root.lock.Unlock()
   762→
   763→	for k := range m.root.children {
   764→		delete(m.root.children, k)
   765→	}
   766→
   767→	return nil
   768→}
   769→
   770→func (m *MemoryStore) SizeInBytes() int64 {
   771→	return m.root.sizeInBytes()
   772→}
   773→
   774→func (m *MemoryStore) SizeInGB() float64 {
   775→	return float64(m.root.sizeInBytes()) / 1e9
   776→}
   777→
   778→// ListChildren , given a selector, returns a list of all children of the level
   779→// selected.
   780→func (m *MemoryStore) ListChildren(selector []string) []string {
   781→	lvl := &m.root
   782→	for lvl != nil && len(selector) != 0 {
   783→		lvl.lock.RLock()
   784→		next := lvl.children[selector[0]]
   785→		lvl.lock.RUnlock()
   786→		lvl = next
   787→		selector = selector[1:]
   788→	}
   789→
   790→	if lvl == nil {
   791→		return nil
   792→	}
   793→
   794→	lvl.lock.RLock()
   795→	defer lvl.lock.RUnlock()
   796→
   797→	children := make([]string, 0, len(lvl.children))
   798→	for child := range lvl.children {
   799→		children = append(children, child)


Found 1 file
pkg/metricstore/checkpoint.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements checkpoint persistence for the in-memory metric store.
     7→//
     8→// Checkpoints enable graceful restarts by periodically saving in-memory metric
     9→// data to disk in JSON or binary format. The checkpoint system:
    10→//
    11→// Key Features:
    12→//   - Periodic background checkpointing via the Checkpointing() worker
    13→//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
    14→//   - Parallel checkpoint creation and loading using worker pools
    15→//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
    16→//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
    17→//   - Only saves unarchived data (archived data is already persisted elsewhere)
    18→//   - GC optimization during loading to prevent excessive heap growth
    19→//
    20→// Checkpoint Workflow:
    21→//  1. Init() loads checkpoints within retention window at startup
    22→//  2. Checkpointing() worker periodically saves new data
    23→//  3. Shutdown() writes final checkpoint before exit
    24→//
    25→// File Organization:
    26→//
    27→//	checkpoints/
    28→//	  cluster1/
    29→//	    host001/
    30→//	      1234567890.json  (JSON format: full subtree snapshot)
    31→//	      1234567890.bin   (binary format: full subtree snapshot)
    32→//	      current.wal      (WAL format: append-only per-entry log)
    33→//	    host002/
    34→//	      ...
    35→package metricstore
    36→
    37→import (
    38→	"bufio"
    39→	"context"
    40→	"encoding/json"
    41→	"errors"
    42→	"fmt"
    43→	"io/fs"
    44→	"os"
    45→	"path"
    46→	"path/filepath"
    47→	"sort"
    48→	"strconv"
    49→	"strings"
    50→	"sync"
    51→	"sync/atomic"
    52→	"time"
    53→
    54→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    55→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    56→)
    57→
    58→const (
    59→	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
    60→	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
    61→	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
    62→)
    63→
    64→// CheckpointMetrics represents metric data in a checkpoint file.
    65→// Whenever the structure changes, update MarshalJSON as well!
    66→type CheckpointMetrics struct {
    67→	Data      []schema.Float `json:"data"`
    68→	Frequency int64          `json:"frequency"`
    69→	Start     int64          `json:"start"`
    70→}
    71→
    72→// CheckpointFile represents the hierarchical structure of a checkpoint file.
    73→// It mirrors the Level tree structure from the MemoryStore.
    74→type CheckpointFile struct {
    75→	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
    76→	Children map[string]*CheckpointFile    `json:"children"`
    77→	From     int64                         `json:"from"`
    78→	To       int64                         `json:"to"`
    79→}
    80→
    81→// lastCheckpoint tracks the timestamp of the last checkpoint creation.
    82→var (
    83→	lastCheckpoint   time.Time
    84→	lastCheckpointMu sync.Mutex
    85→)
    86→
    87→// Checkpointing starts a background worker that periodically saves metric data to disk.
    88→//
    89→// Checkpoints are written every 12 hours (hardcoded).
    90→//
    91→// Format behaviour:
    92→//   - "json": Periodic checkpointing every checkpointInterval
    93→//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
    94→func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
    95→	lastCheckpointMu.Lock()
    96→	lastCheckpoint = time.Now()
    97→	lastCheckpointMu.Unlock()
    98→
    99→	ms := GetMemoryStore()
   100→
   101→	wg.Go(func() {
   102→
   103→		d := 12 * time.Hour // default checkpoint interval
   104→		if Keys.CheckpointInterval != "" {
   105→			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
   106→			if err != nil {
   107→				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
   108→			} else {
   109→				d = parsed
   110→			}
   111→		}
   112→
   113→		ticker := time.NewTicker(d)
   114→		defer ticker.Stop()
   115→
   116→		for {
   117→			select {
   118→			case <-ctx.Done():
   119→				return
   120→			case <-ticker.C:
   121→				lastCheckpointMu.Lock()
   122→				from := lastCheckpoint
   123→				lastCheckpointMu.Unlock()
   124→
   125→				now := time.Now()
   126→				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
   127→
   128→				if Keys.Checkpoints.FileFormat == "wal" {
   129→					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   130→					if err != nil {
   131→						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
   132→					} else {
   133→						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
   134→						lastCheckpointMu.Lock()
   135→						lastCheckpoint = now
   136→						lastCheckpointMu.Unlock()
   137→						// Rotate WAL files for successfully checkpointed hosts.
   138→						RotateWALFiles(hostDirs)
   139→					}
   140→				} else {
   141→					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   142→					if err != nil {
   143→						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
   144→					} else {
   145→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
   146→						lastCheckpointMu.Lock()
   147→						lastCheckpoint = now
   148→						lastCheckpointMu.Unlock()
   149→					}
   150→				}
   151→			}
   152→		}
   153→	})
   154→}
   155→
   156→// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
   157→//
   158→// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
   159→// This method manually constructs JSON to avoid allocations and interface conversions.
   160→func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
   161→	buf := make([]byte, 0, 128+len(cm.Data)*8)
   162→	buf = append(buf, `{"frequency":`...)
   163→	buf = strconv.AppendInt(buf, cm.Frequency, 10)
   164→	buf = append(buf, `,"start":`...)
   165→	buf = strconv.AppendInt(buf, cm.Start, 10)
   166→	buf = append(buf, `,"data":[`...)
   167→	for i, x := range cm.Data {
   168→		if i != 0 {
   169→			buf = append(buf, ',')
   170→		}
   171→		if x.IsNaN() {
   172→			buf = append(buf, `null`...)
   173→		} else {
   174→			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
   175→		}
   176→	}
   177→	buf = append(buf, `]}`...)
   178→	return buf, nil
   179→}
   180→
   181→// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format).
   182→//
   183→// Metrics at root and cluster levels are skipped. One file per host is created.
   184→// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
   185→// at a time, allowing concurrent writes/reads to other hosts.
   186→//
   187→// Returns the number of checkpoint files created and any errors encountered.
   188→func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
   189→	// Pre-calculate capacity by counting cluster/host pairs
   190→	m.root.lock.RLock()
   191→	totalHosts := 0
   192→	for _, l1 := range m.root.children {
   193→		l1.lock.RLock()
   194→		totalHosts += len(l1.children)
   195→		l1.lock.RUnlock()
   196→	}
   197→	m.root.lock.RUnlock()
   198→
   199→	levels := make([]*Level, 0, totalHosts)
   200→	selectors := make([][]string, 0, totalHosts)
   201→
   202→	m.root.lock.RLock()
   203→	for sel1, l1 := range m.root.children {
   204→		l1.lock.RLock()
   205→		for sel2, l2 := range l1.children {
   206→			levels = append(levels, l2)
   207→			selectors = append(selectors, []string{sel1, sel2})
   208→		}
   209→		l1.lock.RUnlock()
   210→	}
   211→	m.root.lock.RUnlock()
   212→
   213→	type workItem struct {
   214→		level    *Level
   215→		dir      string
   216→		selector []string
   217→	}
   218→
   219→	n, errs := int32(0), int32(0)
   220→
   221→	var wg sync.WaitGroup
   222→	wg.Add(Keys.NumWorkers)
   223→	work := make(chan workItem, Keys.NumWorkers*2)
   224→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   225→		go func() {
   226→			defer wg.Done()
   227→
   228→			for workItem := range work {
   229→				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
   230→					if err == ErrNoNewArchiveData {
   231→						continue
   232→					}
   233→
   234→					cclog.Errorf("[METRICSTORE]> error while checkpointing %#v: %s", workItem.selector, err.Error())
   235→					atomic.AddInt32(&errs, 1)
   236→				} else {
   237→					atomic.AddInt32(&n, 1)
   238→				}
   239→			}
   240→		}()
   241→	}
   242→
   243→	for i := 0; i < len(levels); i++ {
   244→		dir := path.Join(dir, path.Join(selectors[i]...))
   245→		work <- workItem{
   246→			level:    levels[i],
   247→			dir:      dir,
   248→			selector: selectors[i],
   249→		}
   250→	}
   251→
   252→	close(work)
   253→	wg.Wait()
   254→
   255→	if errs > 0 {
   256→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   257→	}
   258→	return int(n), nil
   259→}
   260→
   261→// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
   262→// Skips metrics that are already archived. Returns nil if no unarchived data exists.
   263→func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
   264→	l.lock.RLock()
   265→	defer l.lock.RUnlock()
   266→
   267→	retval := &CheckpointFile{
   268→		From:     from,
   269→		To:       to,
   270→		Metrics:  make(map[string]*CheckpointMetrics),
   271→		Children: make(map[string]*CheckpointFile),
   272→	}
   273→
   274→	for metric, minfo := range m.Metrics {
   275→		b := l.metrics[minfo.offset]
   276→		if b == nil {
   277→			continue
   278→		}
   279→
   280→		allArchived := true
   281→		b.iterFromTo(from, to, func(b *buffer) error {
   282→			if !b.archived {
   283→				allArchived = false
   284→				return fmt.Errorf("stop") // Early termination signal
   285→			}
   286→			return nil
   287→		})
   288→
   289→		if allArchived {
   290→			continue
   291→		}
   292→
   293→		data := make([]schema.Float, (to-from)/b.frequency+1)
   294→		data, start, end, err := b.read(from, to, data)
   295→		if err != nil {
   296→			return nil, err
   297→		}
   298→
   299→		for i := int((end - start) / b.frequency); i < len(data); i++ {
   300→			data[i] = schema.NaN
   301→		}
   302→
   303→		retval.Metrics[metric] = &CheckpointMetrics{
   304→			Frequency: b.frequency,
   305→			Start:     start,
   306→			Data:      data,
   307→		}
   308→	}
   309→
   310→	for name, child := range l.children {
   311→		val, err := child.toCheckpointFile(from, to, m)
   312→		if err != nil {
   313→			return nil, err
   314→		}
   315→
   316→		if val != nil {
   317→			retval.Children[name] = val
   318→		}
   319→	}
   320→
   321→	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
   322→		return nil, nil
   323→	}
   324→
   325→	return retval, nil
   326→}
   327→
   328→// toCheckpoint writes a Level's data to a JSON checkpoint file.
   329→// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.
   330→func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
   331→	cf, err := l.toCheckpointFile(from, to, m)
   332→	if err != nil {
   333→		return err
   334→	}
   335→
   336→	if cf == nil {
   337→		return ErrNoNewArchiveData
   338→	}
   339→
   340→	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
   341→	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   342→	if err != nil && os.IsNotExist(err) {
   343→		err = os.MkdirAll(dir, CheckpointDirPerms)
   344→		if err == nil {
   345→			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   346→		}
   347→	}
   348→	if err != nil {
   349→		return err
   350→	}
   351→	defer f.Close()
   352→
   353→	bw := bufio.NewWriter(f)
   354→	if err = json.NewEncoder(bw).Encode(cf); err != nil {
   355→		return err
   356→	}
   357→
   358→	return bw.Flush()
   359→}
   360→
   361→// enqueueCheckpointHosts traverses checkpoint directory and enqueues cluster/host pairs.
   362→// Returns error if directory structure is invalid.
   363→func enqueueCheckpointHosts(dir string, work chan<- [2]string) error {
   364→	clustersDir, err := os.ReadDir(dir)
   365→	if err != nil {
   366→		return err
   367→	}
   368→
   369→	for _, clusterDir := range clustersDir {
   370→		if !clusterDir.IsDir() {
   371→			return errors.New("[METRICSTORE]> expected only directories at first level of checkpoints/ directory")
   372→		}
   373→
   374→		hostsDir, err := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
   375→		if err != nil {
   376→			return err
   377→		}
   378→
   379→		for _, hostDir := range hostsDir {
   380→			if !hostDir.IsDir() {
   381→				return errors.New("[METRICSTORE]> expected only directories at second level of checkpoints/ directory")
   382→			}
   383→
   384→			work <- [2]string{clusterDir.Name(), hostDir.Name()}
   385→		}
   386→	}
   387→
   388→	return nil
   389→}
   390→
   391→// FromCheckpoint loads checkpoint files from disk into memory in parallel.
   392→//
   393→// Uses worker pool to load cluster/host combinations. Returns number of files
   394→// loaded and any errors.
   395→func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
   396→	var wg sync.WaitGroup
   397→	work := make(chan [2]string, Keys.NumWorkers*4)
   398→	n, errs := int32(0), int32(0)
   399→
   400→	wg.Add(Keys.NumWorkers)
   401→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   402→		go func() {
   403→			defer wg.Done()
   404→			for host := range work {
   405→				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
   406→				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
   407→				if err != nil {
   408→					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
   409→					atomic.AddInt32(&errs, 1)
   410→				}
   411→				atomic.AddInt32(&n, int32(nn))
   412→			}
   413→		}()
   414→	}
   415→
   416→	err := enqueueCheckpointHosts(dir, work)
   417→	close(work)
   418→	wg.Wait()
   419→
   420→	if err != nil {
   421→		return int(n), err
   422→	}
   423→
   424→	if errs > 0 {
   425→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   426→	}
   427→	return int(n), nil
   428→}
   429→
   430→// FromCheckpointFiles is the main entry point for loading checkpoints at startup.
   431→//
   432→// Creates checkpoint directory if it doesn't exist. This function must be called
   433→// before any writes or reads, and can only be called once.
   434→func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
   435→	if _, err := os.Stat(dir); os.IsNotExist(err) {
   436→		err := os.MkdirAll(dir, CheckpointDirPerms)
   437→		if err != nil {
   438→			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
   439→		}
   440→		cclog.Debugf("[METRICSTORE]> %#v Directory created successfully", dir)
   441→	}
   442→
   443→	return m.FromCheckpoint(dir, from)
   444→}
   445→
   446→func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
   447→	br := bufio.NewReader(f)
   448→	cf := &CheckpointFile{}
   449→	if err := json.NewDecoder(br).Decode(cf); err != nil {
   450→		return err
   451→	}
   452→
   453→	if cf.To != 0 && cf.To < from {
   454→		return nil
   455→	}
   456→
   457→	if err := l.loadFile(cf, m); err != nil {
   458→		return err
   459→	}
   460→
   461→	return nil
   462→}
   463→
   464→func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
   465→	for name, metric := range cf.Metrics {
   466→		n := len(metric.Data)
   467→		b := &buffer{
   468→			frequency: metric.Frequency,
   469→			start:     metric.Start,
   470→			data:      metric.Data[0:n:n],
   471→			prev:      nil,
   472→			next:      nil,
   473→			archived:  true,
   474→		}
   475→
   476→		minfo, ok := m.Metrics[name]
   477→		if !ok {
   478→			continue
   479→		}
   480→
   481→		prev := l.metrics[minfo.offset]
   482→		if prev == nil {
   483→			l.metrics[minfo.offset] = b
   484→		} else {
   485→			if prev.start > b.start {
   486→				return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start)
   487→			}
   488→
   489→			b.prev = prev
   490→			prev.next = b
   491→		}
   492→		l.metrics[minfo.offset] = b
   493→	}
   494→
   495→	if len(cf.Children) > 0 && l.children == nil {
   496→		l.children = make(map[string]*Level)
   497→	}
   498→
   499→	for sel, childCf := range cf.Children {
   500→		child, ok := l.children[sel]
   501→		if !ok {
   502→			child = &Level{
   503→				metrics:  make([]*buffer, len(m.Metrics)),
   504→				children: nil,
   505→			}
   506→			l.children[sel] = child
   507→		}
   508→
   509→		if err := child.loadFile(childCf, m); err != nil {
   510→			return err
   511→		}
   512→	}
   513→
   514→	return nil
   515→}
   516→
   517→// fromCheckpoint loads all checkpoint files (JSON, binary snapshot, WAL) for a
   518→// single host directory. Snapshot files are loaded first (sorted by timestamp),
   519→// then current.wal is replayed on top.
   520→func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {
   521→	direntries, err := os.ReadDir(dir)
   522→	if err != nil {
   523→		if os.IsNotExist(err) {
   524→			return 0, nil
   525→		}
   526→		return 0, err
   527→	}
   528→
   529→	allFiles := make([]fs.DirEntry, 0)
   530→	var walEntry fs.DirEntry
   531→	filesLoaded := 0
   532→
   533→	for _, e := range direntries {
   534→		if e.IsDir() {
   535→			// Legacy: skip subdirectories (only used by old Avro format).
   536→			// These are ignored; their data is not loaded.
   537→			cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir)
   538→			continue
   539→		}
   540→
   541→		name := e.Name()
   542→		if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") {
   543→			allFiles = append(allFiles, e)
   544→		} else if name == "current.wal" {
   545→			walEntry = e
   546→		}
   547→		// Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes).
   548→	}
   549→
   550→	files, err := findFiles(allFiles, from, true)
   551→	if err != nil {
   552→		return filesLoaded, err
   553→	}
   554→
   555→	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
   556→		".json": l.loadJSONFile,
   557→		".bin":  l.loadBinaryFile,
   558→	}
   559→
   560→	for _, filename := range files {
   561→		ext := filepath.Ext(filename)
   562→		loader := loaders[ext]
   563→		if loader == nil {
   564→			cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename)
   565→			continue
   566→		}
   567→
   568→		err := func() error {
   569→			f, err := os.Open(path.Join(dir, filename))
   570→			if err != nil {
   571→				return err
   572→			}
   573→			defer f.Close()
   574→			return loader(m, f, from)
   575→		}()
   576→		if err != nil {
   577→			return filesLoaded, err
   578→		}
   579→		filesLoaded++
   580→	}
   581→
   582→	// Replay WAL after all snapshot files so it fills in data since the last snapshot.
   583→	if walEntry != nil {
   584→		err := func() error {
   585→			f, err := os.Open(path.Join(dir, walEntry.Name()))
   586→			if err != nil {
   587→				return err
   588→			}
   589→			defer f.Close()
   590→			return l.loadWALFile(m, f, from)
   591→		}()
   592→		if err != nil {
   593→			// WAL errors are non-fatal: the snapshot already loaded the bulk of data.
   594→			cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err)
   595→		} else {
   596→			filesLoaded++
   597→		}
   598→	}
   599→
   600→	return filesLoaded, nil
   601→}
   602→
   603→// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename.
   604→// Supports ".json" (format: "<ts>.json") and ".bin" (format: "<ts>.bin").
   605→func parseTimestampFromFilename(name string) (int64, error) {
   606→	switch {
   607→	case strings.HasSuffix(name, ".json"):
   608→		return strconv.ParseInt(name[:len(name)-5], 10, 64)
   609→	case strings.HasSuffix(name, ".bin"):
   610→		return strconv.ParseInt(name[:len(name)-4], 10, 64)
   611→	default:
   612→		return 0, fmt.Errorf("unknown checkpoint extension for file %q", name)
   613→	}
   614→}
   615→
   616→// findFiles returns filenames from direntries whose timestamps satisfy the filter.
   617→// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the
   618→// last file before t if t falls between two files).
   619→func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {
   620→	nums := map[string]int64{}
   621→	for _, e := range direntries {
   622→		name := e.Name()
   623→		if !strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".bin") {
   624→			continue
   625→		}
   626→
   627→		ts, err := parseTimestampFromFilename(name)
   628→		if err != nil {
   629→			return nil, err
   630→		}
   631→		nums[name] = ts
   632→	}
   633→
   634→	sort.Slice(direntries, func(i, j int) bool {
   635→		a, b := direntries[i], direntries[j]
   636→		return nums[a.Name()] < nums[b.Name()]
   637→	})
   638→
   639→	if len(nums) == 0 {
   640→		return nil, nil
   641→	}
   642→
   643→	filenames := make([]string, 0)
   644→
   645→	for i, e := range direntries {
   646→		ts1 := nums[e.Name()]
   647→
   648→		if findMoreRecentFiles && t <= ts1 {
   649→			filenames = append(filenames, e.Name())
   650→		} else if !findMoreRecentFiles && ts1 <= t && ts1 != 0 {
   651→			filenames = append(filenames, e.Name())
   652→		}
   653→
   654→		if i == len(direntries)-1 {
   655→			continue
   656→		}
   657→
   658→		enext := direntries[i+1]
   659→		ts2 := nums[enext.Name()]
   660→
   661→		if findMoreRecentFiles {
   662→			if ts1 < t && t < ts2 {
   663→				filenames = append(filenames, e.Name())
   664→			}
   665→		}
   666→	}
   667→
   668→	return filenames, nil
   669→}
   670→


Bash

find /Users/jan/prg/ClusterCockpit/cc-backend -type f -name "*.go" -path "*/internal/archiver*" | head -5


/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiver.go
/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go


Found 1 file
internal/archiver/archiveWorker.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package archiver provides asynchronous job archiving functionality for ClusterCockpit.
     7→//
     8→// The archiver runs a background worker goroutine that processes job archiving requests
     9→// from a buffered channel. When jobs complete, their metric data is archived from the
    10→// metric store to the configured archive backend (filesystem, S3, etc.).
    11→//
    12→// # Architecture
    13→//
    14→// The archiver uses a producer-consumer pattern:
    15→//   - Producer: TriggerArchiving() sends jobs to archiveChannel
    16→//   - Consumer: archivingWorker() processes jobs from the channel
    17→//   - Coordination: sync.WaitGroup tracks pending archive operations
    18→//
    19→// # Lifecycle
    20→//
    21→//  1. Start(repo, ctx) - Initialize worker with context for cancellation
    22→//  2. TriggerArchiving(job) - Queue job for archiving (called when job stops)
    23→//  3. archivingWorker() - Background goroutine processes jobs
    24→//  4. Shutdown(timeout) - Graceful shutdown with timeout
    25→//
    26→// # Graceful Shutdown
    27→//
    28→// The archiver supports graceful shutdown with configurable timeout:
    29→//   - Closes channel to reject new jobs
    30→//   - Waits for pending jobs to complete (up to timeout)
    31→//   - Cancels context if timeout exceeded
    32→//   - Ensures worker goroutine exits cleanly
    33→//
    34→// # Example Usage
    35→//
    36→//	// Initialize archiver
    37→//	ctx, cancel := context.WithCancel(context.Background())
    38→//	defer cancel()
    39→//	archiver.Start(jobRepository, ctx)
    40→//
    41→//	// Trigger archiving when job completes
    42→//	archiver.TriggerArchiving(job)
    43→//
    44→//	// Graceful shutdown with 10 second timeout
    45→//	if err := archiver.Shutdown(10 * time.Second); err != nil {
    46→//	    log.Printf("Archiver shutdown timeout: %v", err)
    47→//	}
    48→package archiver
    49→
    50→import (
    51→	"context"
    52→	"fmt"
    53→	"sync"
    54→	"time"
    55→
    56→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    57→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    58→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    59→	sq "github.com/Masterminds/squirrel"
    60→)
    61→
    62→var (
    63→	archivePending sync.WaitGroup
    64→	archiveChannel chan *schema.Job
    65→	jobRepo        *repository.JobRepository
    66→	shutdownCtx    context.Context
    67→	shutdownCancel context.CancelFunc
    68→	workerDone     chan struct{}
    69→)
    70→
    71→// Start initializes the archiver and starts the background worker goroutine.
    72→//
    73→// The archiver processes job archiving requests asynchronously via a buffered channel.
    74→// Jobs are sent to the channel using TriggerArchiving() and processed by the worker.
    75→//
    76→// Parameters:
    77→//   - r: JobRepository instance for database operations
    78→//   - ctx: Context for cancellation (shutdown signal propagation)
    79→//
    80→// The worker goroutine will run until:
    81→//   - ctx is cancelled (via parent shutdown)
    82→//   - archiveChannel is closed (via Shutdown())
    83→//
    84→// Must be called before TriggerArchiving(). Safe to call only once.
    85→func Start(r *repository.JobRepository, ctx context.Context) {
    86→	shutdownCtx, shutdownCancel = context.WithCancel(ctx)
    87→	archiveChannel = make(chan *schema.Job, 128)
    88→	workerDone = make(chan struct{})
    89→	jobRepo = r
    90→
    91→	go archivingWorker()
    92→}
    93→
    94→// archivingWorker is the background goroutine that processes job archiving requests.
    95→//
    96→// The worker loop:
    97→//  1. Blocks waiting for jobs on archiveChannel or shutdown signal
    98→//  2. Fetches job metadata from repository
    99→//  3. Archives job data to configured backend (calls ArchiveJob)
   100→//  4. Updates job footprint and energy metrics in database
   101→//  5. Marks job as successfully archived
   102→//  6. Calls job stop hooks
   103→//
   104→// The worker exits when:
   105→//   - shutdownCtx is cancelled (timeout during shutdown)
   106→//   - archiveChannel is closed (normal shutdown)
   107→//
   108→// Errors during archiving are logged and the job is marked as failed,
   109→// but the worker continues processing other jobs.
   110→func archivingWorker() {
   111→	defer close(workerDone)
   112→
   113→	for {
   114→		select {
   115→		case <-shutdownCtx.Done():
   116→			cclog.Info("Archive worker received shutdown signal")
   117→			return
   118→
   119→		case job, ok := <-archiveChannel:
   120→			if !ok {
   121→				cclog.Info("Archive channel closed, worker exiting")
   122→				return
   123→			}
   124→
   125→			start := time.Now()
   126→			// not using meta data, called to load JobMeta into Cache?
   127→			// will fail if job meta not in repository
   128→			if _, err := jobRepo.FetchMetadata(job); err != nil {
   129→				cclog.Errorf("archiving job (dbid: %d) failed at check metadata step: %s", *job.ID, err.Error())
   130→				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
   131→				archivePending.Done()
   132→				continue
   133→			}
   134→
   135→			// ArchiveJob will fetch all the data from a MetricDataRepository and push into configured archive backend
   136→			// Use shutdown context to allow cancellation
   137→			jobMeta, err := ArchiveJob(job, shutdownCtx)
   138→			if err != nil {
   139→				cclog.Errorf("archiving job (dbid: %d) failed at archiving job step: %s", *job.ID, err.Error())
   140→				jobRepo.UpdateMonitoringStatus(*job.ID, schema.MonitoringStatusArchivingFailed)
   141→				archivePending.Done()
   142→				continue
   143→			}
   144→
   145→			stmt := sq.Update("job").Where("job.id = ?", job.ID)
   146→
   147→			if stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta); err != nil {
   148→				cclog.Errorf("archiving job (dbid: %d) failed at update Footprint step: %s", *job.ID, err.Error())
   149→				archivePending.Done()
   150→				continue
   151→			}
   152→			if stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta); err != nil {
   153→				cclog.Errorf("archiving job (dbid: %d) failed at update Energy step: %s", *job.ID, err.Error())
   154→				archivePending.Done()
   155→				continue
   156→			}
   157→			// Update the jobs database entry one last time:
   158→			stmt = jobRepo.MarkArchived(stmt, schema.MonitoringStatusArchivingSuccessful)
   159→			if err := jobRepo.Execute(stmt); err != nil {
   160→				cclog.Errorf("archiving job (dbid: %d) failed at db execute: %s", *job.ID, err.Error())
   161→				archivePending.Done()
   162→				continue
   163→			}
   164→			cclog.Debugf("archiving job %d took %s", job.JobID, time.Since(start))
   165→			cclog.Infof("archiving job (dbid: %d) successful", *job.ID)
   166→
   167→			repository.CallJobStopHooks(job)
   168→			archivePending.Done()
   169→		}
   170→	}
   171→}
   172→
   173→// TriggerArchiving queues a job for asynchronous archiving.
   174→//
   175→// This function should be called when a job completes (stops) to archive its
   176→// metric data from the metric store to the configured archive backend.
   177→//
   178→// The function:
   179→//  1. Increments the pending job counter (WaitGroup)
   180→//  2. Sends the job to the archiving channel (buffered, capacity 128)
   181→//  3. Returns immediately (non-blocking unless channel is full)
   182→//
   183→// The actual archiving is performed asynchronously by the worker goroutine.
   184→// Upon completion, the worker will decrement the pending counter.
   185→//
   186→// Panics if Start() has not been called first.
   187→func TriggerArchiving(job *schema.Job) {
   188→	if archiveChannel == nil {
   189→		cclog.Fatal("Cannot archive without archiving channel. Did you Start the archiver?")
   190→	}
   191→
   192→	archivePending.Add(1)
   193→	archiveChannel <- job
   194→}
   195→
   196→// Shutdown performs a graceful shutdown of the archiver with a configurable timeout.
   197→//
   198→// The shutdown process:
   199→//  1. Closes archiveChannel - no new jobs will be accepted
   200→//  2. Waits for pending jobs to complete (up to timeout duration)
   201→//  3. If timeout is exceeded:
   202→//     - Cancels shutdownCtx to interrupt ongoing ArchiveJob operations
   203→//     - Returns error indicating timeout
   204→//  4. Waits for worker goroutine to exit cleanly
   205→//
   206→// Parameters:
   207→//   - timeout: Maximum duration to wait for pending jobs to complete
   208→//     (recommended: 10-30 seconds for production)
   209→//
   210→// Returns:
   211→//   - nil if all jobs completed within timeout
   212→//   - error if timeout was exceeded (some jobs may not have been archived)
   213→//
   214→// Jobs that don't complete within the timeout will be marked as failed.
   215→// The function always ensures the worker goroutine exits before returning.
   216→//
   217→// Example:
   218→//
   219→//	if err := archiver.Shutdown(10 * time.Second); err != nil {
   220→//	    log.Printf("Some jobs did not complete: %v", err)
   221→//	}
   222→func Shutdown(timeout time.Duration) error {
   223→	cclog.Info("Initiating archiver shutdown...")
   224→
   225→	// Close channel to signal no more jobs will be accepted
   226→	close(archiveChannel)
   227→
   228→	// Create a channel to signal when all jobs are done
   229→	done := make(chan struct{})
   230→	go func() {
   231→		archivePending.Wait()
   232→		close(done)
   233→	}()
   234→
   235→	// Wait for jobs to complete or timeout
   236→	select {
   237→	case <-done:
   238→		cclog.Info("All archive jobs completed successfully")
   239→		// Wait for worker to exit
   240→		<-workerDone
   241→		return nil
   242→	case <-time.After(timeout):
   243→		cclog.Warn("Archiver shutdown timeout exceeded, cancelling remaining operations")
   244→		// Cancel any ongoing operations
   245→		shutdownCancel()
   246→		// Wait for worker to exit
   247→		<-workerDone
   248→		return fmt.Errorf("archiver shutdown timeout after %v", timeout)
   249→	}
   250→}
   251→


Found 1 file
pkg/metricstore/archive.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"context"
    10→	"errors"
    11→	"fmt"
    12→	"os"
    13→	"path/filepath"
    14→	"strings"
    15→	"sync"
    16→	"sync/atomic"
    17→	"time"
    18→
    19→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    20→)
    21→
    22→// Worker for either Archiving or Deleting files
    23→
    24→func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
    25→	if Keys.Cleanup.Mode == "archive" {
    26→		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
    27→		// Run as Archiver
    28→		cleanUpWorker(wg, ctx,
    29→			Keys.RetentionInMemory,
    30→			"archiving",
    31→			Keys.Cleanup.RootDir,
    32→			false,
    33→		)
    34→	} else {
    35→		// Run as Deleter
    36→		cleanUpWorker(wg, ctx,
    37→			Keys.RetentionInMemory,
    38→			"deleting",
    39→			"",
    40→			true,
    41→		)
    42→	}
    43→}
    44→
    45→// cleanUpWorker takes simple values to configure what it does
    46→func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
    47→	wg.Go(func() {
    48→		d, err := time.ParseDuration(interval)
    49→		if err != nil {
    50→			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
    51→		}
    52→		if d <= 0 {
    53→			return
    54→		}
    55→
    56→		ticker := time.NewTicker(d)
    57→		defer ticker.Stop()
    58→
    59→		for {
    60→			select {
    61→			case <-ctx.Done():
    62→				return
    63→			case <-ticker.C:
    64→				t := time.Now().Add(-d)
    65→				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))
    66→
    67→				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)
    68→
    69→				if err != nil {
    70→					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
    71→				} else {
    72→					if delete {
    73→						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
    74→					} else {
    75→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
    76→					}
    77→				}
    78→			}
    79→		}
    80→	})
    81→}
    82→
    83→var ErrNoNewArchiveData error = errors.New("all data already archived")
    84→
    85→// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
    86→// When archiving, consolidates all hosts per cluster into a single Parquet file.
    87→func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
    88→	if deleteInstead {
    89→		return deleteCheckpoints(checkpointsDir, from)
    90→	}
    91→
    92→	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
    93→}
    94→
    95→// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
    96→func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
    97→	entries1, err := os.ReadDir(checkpointsDir)
    98→	if err != nil {
    99→		return 0, err
   100→	}
   101→
   102→	type workItem struct {
   103→		dir           string
   104→		cluster, host string
   105→	}
   106→
   107→	var wg sync.WaitGroup
   108→	n, errs := int32(0), int32(0)
   109→	work := make(chan workItem, Keys.NumWorkers)
   110→
   111→	wg.Add(Keys.NumWorkers)
   112→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   113→		go func() {
   114→			defer wg.Done()
   115→			for item := range work {
   116→				entries, err := os.ReadDir(item.dir)
   117→				if err != nil {
   118→					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
   119→					atomic.AddInt32(&errs, 1)
   120→					continue
   121→				}
   122→
   123→				files, err := findFiles(entries, from, false)
   124→				if err != nil {
   125→					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
   126→					atomic.AddInt32(&errs, 1)
   127→					continue
   128→				}
   129→
   130→				for _, checkpoint := range files {
   131→					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
   132→						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
   133→						atomic.AddInt32(&errs, 1)
   134→					} else {
   135→						atomic.AddInt32(&n, 1)
   136→					}
   137→				}
   138→			}
   139→		}()
   140→	}
   141→
   142→	for _, de1 := range entries1 {
   143→		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
   144→		if e != nil {
   145→			err = e
   146→			continue
   147→		}
   148→
   149→		for _, de2 := range entries2 {
   150→			work <- workItem{


346:// Server timeout defaults (in seconds)
348:	defaultReadHeaderTimeout = 20
349:	defaultWriteTimeout      = 20
353:	// Use configurable timeouts with defaults
354:	readHeaderTimeout := time.Duration(defaultReadHeaderTimeout) * time.Second
355:	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second
358:		ReadHeaderTimeout: readHeaderTimeout,
359:		WriteTimeout:      writeTimeout,
409:func (s *Server) Shutdown(ctx context.Context) {
410:	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
418:	if err := s.server.Shutdown(shutdownCtx); err != nil {
428:		var wg sync.WaitGroup
432:				metricstore.Shutdown()
437:			if err := archiver.Shutdown(10 * time.Second); err != nil {
442:		wg.Wait()
448:		cclog.Warn("Shutdown deadline exceeded, forcing exit")


   260→// Shutdown performs a graceful shutdown of the metric store.
   261→//
   262→// This function cancels all background goroutines started by Init() and writes
   263→// a final checkpoint to disk before returning. It should be called when the
   264→// application receives a termination signal.
   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→
   290→	ms := GetMemoryStore()
   291→
   292→	lastCheckpointMu.Lock()
   293→	from := lastCheckpoint
   294→	lastCheckpointMu.Unlock()
   295→
   296→	if Keys.Checkpoints.FileFormat == "wal" {
   297→		var hostDirs []string
   298→		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   299→		if err == nil {
   300→			RotateWALFilesAfterShutdown(hostDirs)
   301→		}
   302→	} else {
   303→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   304→	}
   305→
   306→	if err != nil {
   307→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
   308→	}
   309→	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
   310→}
   311→
   312→// Retention starts a background goroutine that periodically frees old metric data.
   313→//
   314→// This worker runs at half the retention interval and calls Free() to remove buffers
   315→// older than the configured retention time. It respects the NodeProvider to preserve
   316→// data for nodes with active jobs.
   317→//
   318→// Parameters:
   319→//   - wg: WaitGroup to signal completion when context is cancelled
   320→//   - ctx: Context for cancellation signal
   321→//
   322→// The goroutine exits when ctx is cancelled.
   323→func Retention(wg *sync.WaitGroup, ctx context.Context) {
   324→	ms := GetMemoryStore()
   325→
   326→	wg.Go(func() {
   327→		d, err := time.ParseDuration(Keys.RetentionInMemory)
   328→		if err != nil {
   329→			cclog.Fatal(err)
   330→		}
   331→		if d <= 0 {
   332→			return
   333→		}
   334→
   335→		tickInterval := d / 2
   336→		if tickInterval <= 0 {
   337→			return
   338→		}
   339→		ticker := time.NewTicker(tickInterval)
   340→		defer ticker.Stop()
   341→
   342→		for {
   343→			select {
   344→			case <-ctx.Done():
   345→				return
   346→			case <-ticker.C:
   347→				state.mu.Lock()
   348→
   349→				t := time.Now().Add(-d)
   350→
   351→				state.lastRetentionTime = t.Unix()
   352→
   353→				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
   354→
   355→				freed, err := Free(ms, t)
   356→				if err != nil {
   357→					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
   358→				} else {
   359→					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)


pkg/metricstore/walCheckpoint.go:627:func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
pkg/metricstore/walCheckpoint.go:706:func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
pkg/metricstore/checkpoint.go:188:func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
pkg/metricstore/checkpoint.go:263:func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
pkg/metricstore/checkpoint.go:330:func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {


   150→// WALStaging starts N sharded background goroutines that receive WALMessage items
   151→// and append binary WAL records to per-host current.wal files.
   152→// Each shard owns a subset of hosts (determined by hash of cluster+node),
   153→// parallelizing serialization and I/O while keeping per-host writes serial.
   154→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   155→	if Keys.Checkpoints.FileFormat == "json" {
   156→		return
   157→	}
   158→
   159→	walNumShards = max(Keys.NumWorkers, 1)
   160→	chBufSize := max(65536/walNumShards, 1024)
   161→
   162→	walShardChs = make([]chan *WALMessage, walNumShards)
   163→	walShardRotateChs = make([]chan walRotateReq, walNumShards)
   164→
   165→	for i := range walNumShards {
   166→		walShardChs[i] = make(chan *WALMessage, chBufSize)
   167→		walShardRotateChs[i] = make(chan walRotateReq, 64)
   168→	}
   169→
   170→	for i := range walNumShards {
   171→		msgCh := walShardChs[i]
   172→		rotateCh := walShardRotateChs[i]
   173→
   174→		wg.Go(func() {
   175→			hostFiles := make(map[string]*walFileState)
   176→
   177→			defer func() {
   178→				for _, ws := range hostFiles {
   179→					if ws.f != nil {
   180→						ws.w.Flush()
   181→						ws.f.Close()
   182→					}
   183→				}
   184→			}()
   185→
   186→			getOrOpenWAL := func(hostDir string) *walFileState {
   187→				ws, ok := hostFiles[hostDir]
   188→				if ok {
   189→					return ws
   190→				}
   191→
   192→				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   193→					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   194→					return nil
   195→				}
   196→
   197→				walPath := path.Join(hostDir, "current.wal")
   198→				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   199→				if err != nil {
   200→					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   201→					return nil
   202→				}
   203→
   204→				w := bufio.NewWriter(f)
   205→
   206→				// Write file header magic if file is new (empty).
   207→				info, err := f.Stat()
   208→				if err == nil && info.Size() == 0 {
   209→					var hdr [4]byte
   210→					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   211→					if _, err := w.Write(hdr[:]); err != nil {
   212→						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   213→						f.Close()
   214→						return nil
   215→					}
   216→				}
   217→
   218→				ws = &walFileState{f: f, w: w}
   219→				hostFiles[hostDir] = ws
   220→				return ws
   221→			}
   222→
   223→			processMsg := func(msg *WALMessage) {
   224→				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   225→				ws := getOrOpenWAL(hostDir)
   226→				if ws == nil {
   227→					return
   228→				}
   229→				if err := writeWALRecordDirect(ws.w, msg); err != nil {
   230→					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   231→				}
   232→				ws.dirty = true
   233→			}
   234→
   235→			processRotate := func(req walRotateReq) {
   236→				ws, ok := hostFiles[req.hostDir]
   237→				if ok && ws.f != nil {
   238→					ws.w.Flush()
   239→					ws.f.Close()
   240→					walPath := path.Join(req.hostDir, "current.wal")
   241→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   242→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   243→					}
   244→					delete(hostFiles, req.hostDir)
   245→				}
   246→				close(req.done)
   247→			}
   248→
   249→			flushDirty := func() {
   250→				for _, ws := range hostFiles {
   251→					if ws.dirty {
   252→						ws.w.Flush()
   253→						ws.dirty = false
   254→					}
   255→				}
   256→			}
   257→
   258→			drain := func() {
   259→				for {
   260→					select {
   261→					case msg, ok := <-msgCh:
   262→						if !ok {
   263→							return
   264→						}
   265→						processMsg(msg)
   266→					case req := <-rotateCh:
   267→						processRotate(req)
   268→					default:
   269→						flushDirty()
   270→						return
   271→					}
   272→				}
   273→			}
   274→
   275→			ticker := time.NewTicker(walFlushInterval)
   276→			defer ticker.Stop()
   277→
   278→			// drainBatch processes up to 4096 pending messages without blocking.
   279→			// Returns false if the channel was closed.
   280→			drainBatch := func() bool {
   281→				for range 4096 {
   282→					select {
   283→					case msg, ok := <-msgCh:
   284→						if !ok {
   285→							flushDirty()
   286→							return false
   287→						}
   288→						processMsg(msg)
   289→					case req := <-rotateCh:
   290→						processRotate(req)
   291→					default:
   292→						return true
   293→					}
   294→				}
   295→				return true
   296→			}
   297→
   298→			for {
   299→				select {
   300→				case <-ctx.Done():
   301→					drain()
   302→					return
   303→				case msg, ok := <-msgCh:
   304→					if !ok {
   305→						return
   306→					}
   307→					processMsg(msg)
   308→					if !drainBatch() {
   309→						return
   310→					}
   311→					// No flush here — timer handles periodic flushing.
   312→				case <-ticker.C:
   313→					flushDirty()
   314→				case req := <-rotateCh:
   315→					processRotate(req)
   316→				}
   317→			}
   318→		})
   319→	}
   320→}
   321→
   322→// RotateWALFiles sends rotation requests for the given host directories
   323→// and blocks until all rotations complete. Each request is routed to the
   324→// shard that owns the host directory.
   325→func RotateWALFiles(hostDirs []string) {
   326→	if walShardRotateChs == nil {
   327→		return
   328→	}
   329→	dones := make([]chan struct{}, len(hostDirs))


   322→// RotateWALFiles sends rotation requests for the given host directories
   323→// and blocks until all rotations complete. Each request is routed to the
   324→// shard that owns the host directory.
   325→func RotateWALFiles(hostDirs []string) {
   326→	if walShardRotateChs == nil {
   327→		return
   328→	}
   329→	dones := make([]chan struct{}, len(hostDirs))
   330→	for i, dir := range hostDirs {
   331→		dones[i] = make(chan struct{})
   332→		// Extract cluster/node from hostDir to find the right shard.
   333→		// hostDir = rootDir/cluster/node
   334→		shard := walShardIndexFromDir(dir)
   335→		walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: dones[i]}
   336→	}
   337→	for _, done := range dones {
   338→		<-done
   339→	}
   340→}
   341→
   342→// walShardIndexFromDir extracts cluster and node from a host directory path
   343→// and returns the shard index. The path format is: rootDir/cluster/node
   344→func walShardIndexFromDir(hostDir string) int {
   345→	// hostDir = .../cluster/node — extract last two path components
   346→	node := path.Base(hostDir)
   347→	cluster := path.Base(path.Dir(hostDir))
   348→	return walShardIndex(cluster, node)
   349→}
   350→
   351→// RotateWALFiles sends rotation requests for the given host directories
   352→// and blocks until all rotations complete.
   353→func RotateWALFilesAfterShutdown(hostDirs []string) {
   354→	for _, dir := range hostDirs {
   355→		walPath := path.Join(dir, "current.wal")
   356→		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   357→			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   358→		}
   359→	}
   360→}
   361→
   362→// writeWALRecordDirect encodes a WAL record directly into the bufio.Writer,
   363→// avoiding heap allocations by using a stack-allocated scratch buffer for
   364→// the fixed-size header/trailer and computing CRC inline.
   365→func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
   366→	// Compute payload size.


Now I have enough information. Let me create a comprehensive summary document of the shutdown flow.

## Shutdown Flow Analysis - 60s Timeout Issue

Based on my thorough exploration of the cc-backend codebase, I've traced the complete shutdown sequence. Here are my findings:

### 1. **Shutdown Signal Handler** (`cmd/cc-backend/main.go:406-418`)

When SIGINT/SIGTERM is received:
```go
go func() {
    select {
    case <-sigs:
        cclog.Info("Shutdown signal received")  // Line 410
    case <-ctx.Done():
    }
    runtime.SystemdNotify(false, "Shutting down ...")
    srv.Shutdown(ctx)
    util.FsWatcherShutdown()
    taskmanager.Shutdown()
}()
```

This calls `srv.Shutdown(ctx)` where `ctx` is the root application context.

---

### 2. **HTTP Server Shutdown** (`cmd/cc-backend/server.go:409-450`)

The `Server.Shutdown()` method has a **60-second timeout window**:

```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)  // 5s for HTTP server
    defer cancel()

    if err := s.server.Shutdown(shutdownCtx); err != nil {
        cclog.Errorf("Server shutdown error: %v", err)
    }

    // Run metricstore and archiver shutdown concurrently (lines 422-443)
    done := make(chan struct{})
    go func() {
        defer close(done)
        var wg sync.WaitGroup

        if ms := metricstore.GetMemoryStore(); ms != nil {
            wg.Go(func() {
                metricstore.Shutdown()  // **BLOCKING OPERATION #1**
            })
        }

        wg.Go(func() {
            if err := archiver.Shutdown(10 * time.Second); err != nil {  // **BLOCKING OPERATION #2**
                cclog.Warnf("Archiver shutdown: %v", err)
            }
        })

        wg.Wait()
    }()

    select {
    case <-done:
        // Success
    case <-time.After(60 * time.Second):  // **60-SECOND TIMEOUT**
        cclog.Warn("Shutdown deadline exceeded, forcing exit")  // Line 448
    }
}
```

---

### 3. **Metricstore Shutdown** (`pkg/metricstore/metricstore.go:273-310`)

This is the primary blocking operation:

```go
func Shutdown() {
    // 1. Cancel all background goroutines
    shutdownFuncMu.Lock()
    if shutdownFunc != nil {
        shutdownFunc()  // Cancels context
    }
    shutdownFuncMu.Unlock()

    // 2. Close WAL channels (if WAL format is enabled)
    if Keys.Checkpoints.FileFormat == "wal" {
        for _, ch := range walShardChs {
            close(ch)  // Signal WAL staging goroutines to stop
        }
    }

    cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)

    // 3. **WRITES FINAL CHECKPOINT** - This can be very slow
    if Keys.Checkpoints.FileFormat == "wal" {
        // Binary snapshot format
        files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
        if err == nil {
            RotateWALFilesAfterShutdown(hostDirs)
        }
    } else {
        // JSON format
        files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
    }
}
```

**Key Performance Issues:**

a) **Checkpoint Writing** (`pkg/metricstore/checkpoint.go:188-259` or `walCheckpoint.go:627-702`):
   - Uses worker pool (NumWorkers threads) to write checkpoint files in parallel
   - Each host directory produces ONE checkpoint file
   - For WAL format: writes binary snapshots for all hosts, then rotates WAL files
   - For JSON format: writes JSON files for all hosts
   - Worker pool uses buffered channels with up to `NumWorkers*2` items
   - **Potential slowdown**: Large memory store with many hosts means many file I/O operations

b) **Lock Contention During Checkpoint** (`pkg/metricstore/checkpoint.go:263-326`):
   - `toCheckpointFile()` holds `RLock` while traversing the entire level tree
   - Reads all metrics data from buffers and children recursively
   - This can block concurrent reads/writes during shutdown

c) **WAL Rotation** (`pkg/metricstore/walCheckpoint.go:353-360`):
   - `RotateWALFilesAfterShutdown()` simply deletes WAL files (non-blocking)
   - But if using `RotateWALFiles()` before shutdown, it **blocks** until all rotation requests complete (lines 325-340)

---

### 4. **Archiver Shutdown** (`internal/archiver/archiveWorker.go:222-250`)

```go
func Shutdown(timeout time.Duration) error {
    cclog.Info("Initiating archiver shutdown...")

    // 1. Close channel - reject new jobs
    close(archiveChannel)

    // 2. Wait for pending jobs to complete
    done := make(chan struct{})
    go func() {
        archivePending.Wait()  // Waits for all archiving operations
        close(done)
    }()

    // 3. Wait with timeout (10 seconds)
    select {
    case <-done:
        cclog.Info("All archive jobs completed successfully")
        <-workerDone  // Wait for worker goroutine
        return nil
    case <-time.After(timeout):  // 10-second timeout
        cclog.Warn("Archiver shutdown timeout exceeded, cancelling remaining operations")
        shutdownCancel()  // Cancel ongoing ArchiveJob operations
        <-workerDone     // Wait for worker to exit
        return fmt.Errorf("archiver shutdown timeout after %v", timeout)
    }
}
```

**Potential Blocking Points:**
- If jobs are being archived and take > 10 seconds, they get cancelled
- `ArchiveJob()` fetches metrics from the metric store - can be slow for large jobs

---

### 5. **Background Goroutines That Must Exit**

Before `Shutdown()` returns, these must all be cancelled (via context):

1. **Retention Worker** (`pkg/metricstore/metricstore.go:323-369`): Periodically frees old buffers
2. **Checkpointing Worker** (`pkg/metricstore/checkpoint.go:94-154`): Periodic checkpointing every 12 hours
3. **CleanUp Worker** (`pkg/metricstore/archive.go:24-43`): Deletes/archives old checkpoint files
4. **WALStaging Workers** (`pkg/metricstore/walCheckpoint.go:154-320`): N sharded goroutines writing WAL records
5. **MemoryUsageTracker** (`pkg/metricstore/metricstore.go:386-492`): Monitors memory and emergency frees buffers
6. **NATS Receiver** (`pkg/metricstore/metricstore.go:187-192`): Optionally subscribes to NATS messages

These goroutines must respond to `ctx.Done()` within milliseconds.

---

### 6. **Timeline of 60-Second Timeout Sequence**

```
T+0.0s:  Signal handler calls srv.Shutdown(ctx)
T+0.0s:  HTTP server shutdown starts (5s timeout)
T+0.0-5s: Accept no new connections, drain existing

T+0.0s:  metricstore.Shutdown() and archiver.Shutdown() run in parallel
         - Background goroutines cancelled immediately
         - WAL staging channels closed

T+0.0-?s: metricstore.Shutdown() blocked on ToCheckpoint/ToCheckpointWAL
         - Writing final checkpoint(s) to disk
         - Can involve heavy I/O and CPU (parallel workers)
         - **DEPENDS ON**:
           - Total metric data size
           - Number of hosts/clusters
           - Disk I/O latency
           - CPU contention with concurrent operations

T+0.0-10s: archiver.Shutdown() waits for pending archive jobs
         - Jobs queued in archiveChannel still being processed
         - **DEPENDS ON**:
           - Number of pending jobs in queue
           - Time each job takes to archive
           - Metric store query performance during checkpoint

T+60s:   **TIMEOUT EXCEEDED** - "Shutdown deadline exceeded, forcing exit"
         - Forcefully terminates remaining operations
         - May leave partial checkpoint files
         - WAL files may not be rotated
         - Pending archive jobs may not complete
```

---

### 7. **Key Performance Factors Contributing to 60s Timeout**

1. **Checkpoint File Size**: If you have many hosts/clusters or high sampling rates, the final checkpoint can be very large
2. **Disk I/O**: Network storage (NFS) or slow SSDs will make `ToCheckpoint()` slower
3. **NumWorkers Setting**: Too few workers means serialized file writes; too many causes contention
4. **Pending Archive Jobs**: Large unarchived jobs at shutdown time will consume the archiver timeout
5. **Memory Store Size**: Larger in-memory data = longer to serialize to checkpoint files
6. **Lock Contention**: If other operations (reads, writes) continue during shutdown, checkpoint readers block

---

### 8. **Why WAL Can Be Slower Than JSON**

**WAL Format** (`checkpoint.go` + `walCheckpoint.go`):
- Calls `ms.ToCheckpointWAL()` (binary snapshot)
- Writes binary files for each host (potentially more data per file)
- Then calls `RotateWALFilesAfterShutdown()` to delete WAL files
- Closes all WAL shard channels and waits for channels to drain

**JSON Format** (`checkpoint.go` only):
- Calls `ms.ToCheckpoint()`
- Writes JSON files using buffered I/O with `json.Encoder`
- Generally faster serialization than binary format
- No WAL channels to close

---

### 9. **Files and Functions Involved**

| File | Function | Role | Type |
|------|----------|------|------|
| `cmd/cc-backend/main.go` | `runServer()` lines 339-463 | Signal handler setup | Entry point |
| `cmd/cc-backend/server.go` | `Shutdown()` lines 409-450 | Orchestrates shutdown, **has 60s timeout** | **CRITICAL** |
| `pkg/metricstore/metricstore.go` | `Shutdown()` lines 273-310 | Cancels workers, writes final checkpoint | **BLOCKING** |
| `pkg/metricstore/checkpoint.go` | `ToCheckpoint()` lines 188-259 | Writes JSON checkpoints in parallel | **I/O INTENSIVE** |
| `pkg/metricstore/walCheckpoint.go` | `ToCheckpointWAL()` lines 627-702 | Writes binary snapshots in parallel | **I/O INTENSIVE** |
| `pkg/metricstore/walCheckpoint.go` | `WALStaging()` lines 154-320 | N sharded WAL writer goroutines | Background |
| `internal/archiver/archiveWorker.go` | `Shutdown()` lines 222-250 | Waits for archiving queue, **10s timeout** | **BLOCKING** |

---

### 10. **Potential Solutions**

1. **Increase 60s timeout** in `server.go:447` to allow more time
2. **Reduce checkpoint data** by shortening retention window
3. **Optimize disk I/O** (faster storage, better configuration)
4. **Increase NumWorkers** if CPU-bound, decrease if I/O-bound
5. **Monitor pending archive jobs** at shutdown time
6. **Profile shutdown** to identify which operation is actually slow
7. **Consider skipping final checkpoint** if system is already backed up elsewhere
8. **Lower archiver timeout** (currently 10s) if you don't need to wait for in-flight jobs

The **root cause** is almost certainly the final checkpoint write (`ToCheckpoint` or `ToCheckpointWAL`), which must serialize and write all in-memory metric data to disk before the 60-second deadline.

Prompt:
I need to understand the full shutdown sequence of cc-backend. The user sees a 60s timeout at shutdown. Trace the shutdown flow thoroughly:

1. Find the shutdown signal handler in cmd/cc-backend/main.go (around line 410)
2. Read the full shutdown sequence - what happens after the signal is received
3. Find server.go around line 448 where "Shutdown deadline exceeded" is logged
4. Find metricstore.go around line 286 where checkpoint writing happens
5. Look at the WAL checkpoint code - what does Shutdown() do in metricstore?
6. Look at walCheckpoint.go for WAL-related shutdown/cleanup
7. Check if WAL files are deleted during shutdown
8. Find all goroutines or blocking operations that could delay shutdown

Be very thorough - read all relevant files completely. I need to understand every step between "Shutdown signal received" and the timeout.


Now let me read the key files directly to understand the exact code.


   405→	}
   406→	return nil
   407→}
   408→
   409→func (s *Server) Shutdown(ctx context.Context) {
   410→	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
   411→	defer cancel()
   412→
   413→	nc := nats.GetClient()
   414→	if nc != nil {
   415→		nc.Close()
   416→	}
   417→
   418→	if err := s.server.Shutdown(shutdownCtx); err != nil {
   419→		cclog.Errorf("Server shutdown error: %v", err)
   420→	}
   421→
   422→	// Run metricstore and archiver shutdown concurrently.
   423→	// They are independent: metricstore writes .bin snapshots,
   424→	// archiver flushes pending job archives.
   425→	done := make(chan struct{})
   426→	go func() {
   427→		defer close(done)
   428→		var wg sync.WaitGroup
   429→
   430→		if ms := metricstore.GetMemoryStore(); ms != nil {
   431→			wg.Go(func() {
   432→				metricstore.Shutdown()
   433→			})
   434→		}
   435→
   436→		wg.Go(func() {
   437→			if err := archiver.Shutdown(10 * time.Second); err != nil {
   438→				cclog.Warnf("Archiver shutdown: %v", err)
   439→			}
   440→		})
   441→
   442→		wg.Wait()
   443→	}()
   444→
   445→	select {
   446→	case <-done:
   447→	case <-time.After(60 * time.Second):
   448→		cclog.Warn("Shutdown deadline exceeded, forcing exit")
   449→	}
   450→}
   451→

	}
	return nil
}

func (s *Server) Shutdown(ctx context.Context) {
	shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Run metricstore and archiver shutdown concurrently.
	// They are independent: metricstore writes .bin snapshots,
	// archiver flushes pending job archives.
	done := make(chan struct{})
	go func() {
		defer close(done)
		var wg sync.WaitGroup

		if ms := metricstore.GetMemoryStore(); ms != nil {
			wg.Go(func() {
				metricstore.Shutdown()
			})
		}

		wg.Go(func() {
			if err := archiver.Shutdown(10 * time.Second); err != nil {
				cclog.Warnf("Archiver shutdown: %v", err)
			}
		})

		wg.Wait()
	}()

	select {
	case <-done:
	case <-time.After(60 * time.Second):
		cclog.Warn("Shutdown deadline exceeded, forcing exit")
	}
}


   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→
   290→	ms := GetMemoryStore()
   291→
   292→	lastCheckpointMu.Lock()
   293→	from := lastCheckpoint
   294→	lastCheckpointMu.Unlock()
   295→
   296→	if Keys.Checkpoints.FileFormat == "wal" {
   297→		var hostDirs []string
   298→		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   299→		if err == nil {
   300→			RotateWALFilesAfterShutdown(hostDirs)
   301→		}
   302→	} else {
   303→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   304→	}
   305→
   306→	if err != nil {
   307→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
   308→	}
   309→	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
   310→}
   311→
   312→// Retention starts a background goroutine that periodically frees old metric data.
   313→//
   314→// This worker runs at half the retention interval and calls Free() to remove buffers
   315→// older than the configured retention time. It respects the NodeProvider to preserve
   316→// data for nodes with active jobs.
   317→//
   318→// Parameters:
   319→//   - wg: WaitGroup to signal completion when context is cancelled

//
// The function will:
//  1. Cancel the context to stop all background workers
//  2. Close the WAL messages channel if using WAL format
//  3. Write a final checkpoint to preserve in-memory data
//  4. Log any errors encountered during shutdown
//
// Note: This function blocks until the final checkpoint is written.
func Shutdown() {
	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}

	if Keys.Checkpoints.FileFormat == "wal" {
		for _, ch := range walShardChs {
			close(ch)
		}
	}

	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		var hostDirs []string
		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
		if err == nil {
			RotateWALFilesAfterShutdown(hostDirs)
		}
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
}

// Retention starts a background goroutine that periodically frees old metric data.
//
// This worker runs at half the retention interval and calls Free() to remove buffers
// older than the configured retention time. It respects the NodeProvider to preserve
// data for nodes with active jobs.
//
// Parameters:
//   - wg: WaitGroup to signal completion when context is cancelled


   620→	}
   621→	return result.String()
   622→}
   623→
   624→// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
   625→// Returns the number of files written, the list of host directories that were
   626→// successfully checkpointed (for WAL rotation), and any errors.
   627→func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
   628→	// Collect all cluster/host pairs.
   629→	m.root.lock.RLock()
   630→	totalHosts := 0
   631→	for _, l1 := range m.root.children {
   632→		l1.lock.RLock()
   633→		totalHosts += len(l1.children)
   634→		l1.lock.RUnlock()
   635→	}
   636→	m.root.lock.RUnlock()
   637→
   638→	levels := make([]*Level, 0, totalHosts)
   639→	selectors := make([][]string, 0, totalHosts)
   640→
   641→	m.root.lock.RLock()
   642→	for sel1, l1 := range m.root.children {
   643→		l1.lock.RLock()
   644→		for sel2, l2 := range l1.children {
   645→			levels = append(levels, l2)
   646→			selectors = append(selectors, []string{sel1, sel2})
   647→		}
   648→		l1.lock.RUnlock()
   649→	}
   650→	m.root.lock.RUnlock()
   651→
   652→	type workItem struct {
   653→		level    *Level
   654→		hostDir  string
   655→		selector []string
   656→	}
   657→
   658→	n, errs := int32(0), int32(0)
   659→	var successDirs []string
   660→	var successMu sync.Mutex
   661→
   662→	var wg sync.WaitGroup
   663→	wg.Add(Keys.NumWorkers)
   664→	work := make(chan workItem, Keys.NumWorkers*2)
   665→
   666→	for range Keys.NumWorkers {
   667→		go func() {
   668→			defer wg.Done()
   669→			for wi := range work {
   670→				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
   671→				if err != nil {
   672→					if err == ErrNoNewArchiveData {
   673→						continue
   674→					}
   675→					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
   676→					atomic.AddInt32(&errs, 1)
   677→				} else {
   678→					atomic.AddInt32(&n, 1)
   679→					successMu.Lock()
   680→					successDirs = append(successDirs, wi.hostDir)
   681→					successMu.Unlock()
   682→				}
   683→			}
   684→		}()
   685→	}
   686→
   687→	for i := range levels {
   688→		hostDir := path.Join(dir, path.Join(selectors[i]...))
   689→		work <- workItem{
   690→			level:    levels[i],
   691→			hostDir:  hostDir,
   692→			selector: selectors[i],
   693→		}
   694→	}
   695→	close(work)
   696→	wg.Wait()
   697→
   698→	if errs > 0 {
   699→		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
   700→	}
   701→	return int(n), successDirs, nil
   702→}
   703→
   704→// toCheckpointBinary writes a binary snapshot file for a single host-level node.
   705→// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
   706→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   707→	cf, err := l.toCheckpointFile(from, to, m)
   708→	if err != nil {
   709→		return err

	}
	return result.String()
}

// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
// Returns the number of files written, the list of host directories that were
// successfully checkpointed (for WAL rotation), and any errors.
func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
	// Collect all cluster/host pairs.
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		hostDir  string
		selector []string
	}

	n, errs := int32(0), int32(0)
	var successDirs []string
	var successMu sync.Mutex

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)

	for range Keys.NumWorkers {
		go func() {
			defer wg.Done()
			for wi := range work {
				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
				if err != nil {
					if err == ErrNoNewArchiveData {
						continue
					}
					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
					successMu.Lock()
					successDirs = append(successDirs, wi.hostDir)
					successMu.Unlock()
				}
			}
		}()
	}

	for i := range levels {
		hostDir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			hostDir:  hostDir,
			selector: selectors[i],
		}
	}
	close(work)
	wg.Wait()

	if errs > 0 {
		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
	}
	return int(n), successDirs, nil
}

// toCheckpointBinary writes a binary snapshot file for a single host-level node.
// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err


   340→}
   341→
   342→// walShardIndexFromDir extracts cluster and node from a host directory path
   343→// and returns the shard index. The path format is: rootDir/cluster/node
   344→func walShardIndexFromDir(hostDir string) int {
   345→	// hostDir = .../cluster/node — extract last two path components
   346→	node := path.Base(hostDir)
   347→	cluster := path.Base(path.Dir(hostDir))
   348→	return walShardIndex(cluster, node)
   349→}
   350→
   351→// RotateWALFiles sends rotation requests for the given host directories
   352→// and blocks until all rotations complete.
   353→func RotateWALFilesAfterShutdown(hostDirs []string) {
   354→	for _, dir := range hostDirs {
   355→		walPath := path.Join(dir, "current.wal")
   356→		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   357→			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   358→		}
   359→	}
   360→}
   361→
   362→// writeWALRecordDirect encodes a WAL record directly into the bufio.Writer,
   363→// avoiding heap allocations by using a stack-allocated scratch buffer for
   364→// the fixed-size header/trailer and computing CRC inline.
   365→func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
   366→	// Compute payload size.
   367→	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
   368→	for _, s := range msg.Selector {
   369→		payloadSize += 1 + len(s)

}

// walShardIndexFromDir extracts cluster and node from a host directory path
// and returns the shard index. The path format is: rootDir/cluster/node
func walShardIndexFromDir(hostDir string) int {
	// hostDir = .../cluster/node — extract last two path components
	node := path.Base(hostDir)
	cluster := path.Base(path.Dir(hostDir))
	return walShardIndex(cluster, node)
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete.
func RotateWALFilesAfterShutdown(hostDirs []string) {
	for _, dir := range hostDirs {
		walPath := path.Join(dir, "current.wal")
		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
		}
	}
}

// writeWALRecordDirect encodes a WAL record directly into the bufio.Writer,
// avoiding heap allocations by using a stack-allocated scratch buffer for
// the fixed-size header/trailer and computing CRC inline.
func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
	// Compute payload size.
	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
	for _, s := range msg.Selector {
		payloadSize += 1 + len(s)