     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package taskmanager
     7→
     8→import (
     9→	"bytes"
    10→	"encoding/json"
    11→	"time"
    12→
    13→	"github.com/ClusterCockpit/cc-backend/internal/auth"
    14→	"github.com/ClusterCockpit/cc-backend/internal/config"
    15→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    16→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    17→	"github.com/go-co-op/gocron/v2"
    18→)
    19→
    20→const (
    21→	DefaultCompressOlderThan = 7
    22→)
    23→
    24→// Retention defines the configuration for job retention policies.
    25→type Retention struct {
    26→	Policy             string `json:"policy"`
    27→	Format             string `json:"format"`
    28→	Age                int    `json:"age"`
    29→	IncludeDB          bool   `json:"include-db"`
    30→	OmitTagged         string `json:"omit-tagged"`
    31→	TargetKind         string `json:"target-kind"`
    32→	TargetPath         string `json:"target-path"`
    33→	TargetEndpoint     string `json:"target-endpoint"`
    34→	TargetBucket       string `json:"target-bucket"`
    35→	TargetAccessKey    string `json:"target-access-key"`
    36→	TargetSecretKey    string `json:"target-secret-key"`
    37→	TargetRegion       string `json:"target-region"`
    38→	TargetUsePathStyle bool   `json:"target-use-path-style"`
    39→	MaxFileSizeMB      int    `json:"max-file-size-mb"`
    40→}
    41→
    42→// CronFrequency defines the execution intervals for various background workers.
    43→type CronFrequency struct {
    44→	// Duration Update Worker [Defaults to '2m']
    45→	CommitJobWorker string `json:"commit-job-worker"`
    46→	// Duration Update Worker [Defaults to '5m']
    47→	DurationWorker string `json:"duration-worker"`
    48→	// Metric-Footprint Update Worker [Defaults to '10m']
    49→	FootprintWorker string `json:"footprint-worker"`
    50→}
    51→
    52→var (
    53→	s       gocron.Scheduler
    54→	jobRepo *repository.JobRepository
    55→	// Keys holds the configured frequencies for cron jobs.
    56→	Keys CronFrequency
    57→)
    58→
    59→// parseDuration parses a duration string and handles errors by logging them.
    60→// It returns the duration and any error encountered.
    61→func parseDuration(s string) (time.Duration, error) {
    62→	interval, err := time.ParseDuration(s)
    63→	if err != nil {
    64→		cclog.Warnf("Could not parse duration for sync interval: %v",
    65→			s)
    66→		return 0, err
    67→	}
    68→
    69→	if interval == 0 {
    70→		cclog.Info("TaskManager: Sync interval is zero")
    71→	}
    72→
    73→	return interval, nil
    74→}
    75→
    76→func initArchiveServices(config json.RawMessage) {
    77→	var cfg struct {
    78→		Retention   Retention `json:"retention"`
    79→		Compression int       `json:"compression"`
    80→	}
    81→	cfg.Retention.IncludeDB = true
    82→
    83→	if err := json.Unmarshal(config, &cfg); err != nil {
    84→		cclog.Errorf("error while unmarshaling raw config json: %v", err)
    85→	}
    86→
    87→	switch cfg.Retention.Policy {
    88→	case "delete":
    89→		RegisterRetentionDeleteService(cfg.Retention)
    90→	case "copy":
    91→		RegisterRetentionCopyService(cfg.Retention)
    92→	case "move":
    93→		RegisterRetentionMoveService(cfg.Retention)
    94→	}
    95→
    96→	if cfg.Compression > 0 {
    97→		RegisterCompressionService(cfg.Compression)
    98→	} else {
    99→		RegisterCompressionService(DefaultCompressOlderThan)
   100→	}
   101→}
   102→
   103→// Start initializes the task manager, parses configurations, and registers background tasks.
   104→// It starts the gocron scheduler.
   105→func Start(cronCfg, archiveConfig json.RawMessage) {
   106→	var err error
   107→	jobRepo = repository.GetJobRepository()
   108→	s, err = gocron.NewScheduler()
   109→	if err != nil {
   110→		cclog.Abortf("Taskmanager Start: Could not create gocron scheduler.\nError: %s\n", err.Error())
   111→	}
   112→
   113→	if config.Keys.StopJobsExceedingWalltime > 0 {
   114→		RegisterStopJobsExceedTime()
   115→	}
   116→
   117→	dec := json.NewDecoder(bytes.NewReader(cronCfg))
   118→	dec.DisallowUnknownFields()
   119→	if err := dec.Decode(&Keys); err != nil {
   120→		cclog.Errorf("error while decoding cron config: %v", err)
   121→	}
   122→
   123→	if archiveConfig != nil {
   124→		initArchiveServices(archiveConfig)
   125→	} else {
   126→		// Always enable compression
   127→		RegisterCompressionService(DefaultCompressOlderThan)
   128→	}
   129→
   130→	lc := auth.Keys.LdapConfig
   131→
   132→	if lc != nil && lc.SyncInterval != "" {
   133→		RegisterLdapSyncService(lc.SyncInterval)
   134→	}
   135→
   136→	RegisterFootprintWorker()
   137→	RegisterUpdateDurationWorker()
   138→	RegisterCommitJobService()
   139→
   140→	if config.Keys.NodeStateRetention != nil && config.Keys.NodeStateRetention.Policy != "" {
   141→		initNodeStateRetention()
   142→	}
   143→
   144→	s.Start()
   145→}
   146→
   147→func initNodeStateRetention() {
   148→	cfg := config.Keys.NodeStateRetention
   149→	age := cfg.Age
   150→	if age <= 0 {
   151→		age = 24
   152→	}
   153→
   154→	switch cfg.Policy {
   155→	case "delete":
   156→		RegisterNodeStateRetentionDeleteService(age)
   157→	case "move":
   158→		RegisterNodeStateRetentionMoveService(cfg)
   159→	default:
   160→		cclog.Warnf("Unknown nodestate-retention policy: %s", cfg.Policy)
   161→	}
   162→}
   163→
   164→// Shutdown stops the task manager and its scheduler.
   165→func Shutdown() {
   166→	if s != nil {
   167→		s.Shutdown()
   168→	}
   169→}
   170→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides walCheckpoint.go: WAL-based checkpoint implementation.
     7→//
     8→// This replaces the Avro shadow tree with an append-only Write-Ahead Log (WAL)
     9→// per host, eliminating the extra memory overhead of the AvroStore and providing
    10→// truly continuous (per-write) crash safety.
    11→//
    12→// # Architecture
    13→//
    14→//	Metric write (DecodeLine)
    15→//	      │
    16→//	      ├─► WriteToLevel()    → main MemoryStore (unchanged)
    17→//	      │
    18→//	      └─► WALMessages channel
    19→//	               │
    20→//	               ▼
    21→//	        WALStaging goroutine
    22→//	               │
    23→//	               ▼
    24→//	        checkpoints/cluster/host/current.wal  (append-only, binary)
    25→//
    26→//	Periodic checkpoint (Checkpointing goroutine):
    27→//	  1. Write <timestamp>.bin snapshot (column-oriented, from main tree)
    28→//	  2. Signal WALStaging to truncate current.wal per host
    29→//
    30→//	On restart (FromCheckpoint):
    31→//	  1. Load most recent <timestamp>.bin snapshot
    32→//	  2. Replay current.wal (overwrite-safe: buffer.write handles duplicate timestamps)
    33→//
    34→// # WAL Record Format
    35→//
    36→//	[4B magic 0xCC1DA7A1][4B payload_len][payload][4B CRC32]
    37→//
    38→//	payload:
    39→//	  [8B timestamp int64]
    40→//	  [2B metric_name_len uint16][N metric name bytes]
    41→//	  [1B selector_count uint8]
    42→//	  per selector: [1B selector_len uint8][M selector bytes]
    43→//	  [4B value float32 bits]
    44→//
    45→// # Binary Snapshot Format
    46→//
    47→//	[4B magic 0xCC5B0001][8B from int64][8B to int64]
    48→//	Level tree (recursive):
    49→//	  [4B num_metrics uint32]
    50→//	  per metric:
    51→//	    [2B name_len uint16][N name bytes]
    52→//	    [8B frequency int64][8B start int64]
    53→//	    [4B num_values uint32][num_values × 4B float32]
    54→//	  [4B num_children uint32]
    55→//	  per child: [2B name_len uint16][N name bytes] + Level (recursive)
    56→package metricstore
    57→
    58→import (
    59→	"bufio"
    60→	"context"
    61→	"encoding/binary"
    62→	"fmt"
    63→	"hash/crc32"
    64→	"hash/fnv"
    65→	"io"
    66→	"math"
    67→	"os"
    68→	"path"
    69→	"strings"
    70→	"sync"
    71→	"sync/atomic"
    72→	"time"
    73→
    74→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    75→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    76→)
    77→
    78→// Magic numbers for binary formats.
    79→const (
    80→	walFileMagic   = uint32(0xCC1DA701) // WAL file header magic
    81→	walRecordMagic = uint32(0xCC1DA7A1) // WAL record magic
    82→	snapFileMagic  = uint32(0xCC5B0001) // Binary snapshot magic
    83→)
    84→
    85→// walShardChs holds per-shard channels for WAL messages.
    86→// Initialized by WALStaging; nil when WAL is not active.
    87→var walShardChs []chan *WALMessage
    88→
    89→// walShardRotateChs holds per-shard channels for WAL rotation requests.
    90→var walShardRotateChs []chan walRotateReq
    91→
    92→// walNumShards stores the number of shards (set during WALStaging init).
    93→var walNumShards int
    94→
    95→// WALMessage represents a single metric write to be appended to the WAL.
    96→// Cluster and Node are NOT stored in the WAL record (inferred from file path).
    97→type WALMessage struct {
    98→	MetricName string
    99→	Cluster    string
   100→	Node       string
   101→	Selector   []string
   102→	Value      schema.Float
   103→	Timestamp  int64
   104→}
   105→
   106→// walRotateReq requests WAL file rotation for a specific host directory.
   107→// The done channel is closed by the WAL goroutine when rotation is complete.
   108→type walRotateReq struct {
   109→	hostDir string
   110→	done    chan struct{}
   111→}
   112→
   113→// walFileState holds an open WAL file handle and buffered writer for one host directory.
   114→type walFileState struct {
   115→	f     *os.File
   116→	w     *bufio.Writer
   117→	dirty bool
   118→}
   119→
   120→// walFlushInterval controls how often dirty WAL files are flushed to disk.
   121→// Decoupling flushes from message processing lets the consumer run at memory
   122→// speed, amortizing syscall overhead across many writes.
   123→const walFlushInterval = 5 * time.Second
   124→
   125→// walShardIndex computes which shard a message belongs to based on cluster+node.
   126→// Uses FNV-1a hash for fast, well-distributed mapping.
   127→func walShardIndex(cluster, node string) int {
   128→	h := fnv.New32a()
   129→	h.Write([]byte(cluster))
   130→	h.Write([]byte{0})
   131→	h.Write([]byte(node))
   132→	return int(h.Sum32()) % walNumShards
   133→}
   134→
   135→// SendWALMessage routes a WAL message to the appropriate shard channel.
   136→// Returns false if the channel is full (message dropped).
   137→func SendWALMessage(msg *WALMessage) bool {
   138→	if walShardChs == nil {
   139→		return false
   140→	}
   141→	shard := walShardIndex(msg.Cluster, msg.Node)
   142→	select {
   143→	case walShardChs[shard] <- msg:
   144→		return true
   145→	default:
   146→		return false
   147→	}
   148→}
   149→
   150→// WALStaging starts N sharded background goroutines that receive WALMessage items
   151→// and append binary WAL records to per-host current.wal files.
   152→// Each shard owns a subset of hosts (determined by hash of cluster+node),
   153→// parallelizing serialization and I/O while keeping per-host writes serial.
   154→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   155→	if Keys.Checkpoints.FileFormat == "json" {
   156→		return
   157→	}
   158→
   159→	walNumShards = max(Keys.NumWorkers, 1)
   160→	chBufSize := max(65536/walNumShards, 1024)
   161→
   162→	walShardChs = make([]chan *WALMessage, walNumShards)
   163→	walShardRotateChs = make([]chan walRotateReq, walNumShards)
   164→
   165→	for i := range walNumShards {
   166→		walShardChs[i] = make(chan *WALMessage, chBufSize)
   167→		walShardRotateChs[i] = make(chan walRotateReq, 64)
   168→	}
   169→
   170→	for i := range walNumShards {
   171→		msgCh := walShardChs[i]
   172→		rotateCh := walShardRotateChs[i]
   173→
   174→		wg.Go(func() {
   175→			hostFiles := make(map[string]*walFileState)
   176→
   177→			defer func() {
   178→				for _, ws := range hostFiles {
   179→					if ws.f != nil {
   180→						ws.w.Flush()
   181→						ws.f.Close()
   182→					}
   183→				}
   184→			}()
   185→
   186→			getOrOpenWAL := func(hostDir string) *walFileState {
   187→				ws, ok := hostFiles[hostDir]
   188→				if ok {
   189→					return ws
   190→				}
   191→
   192→				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   193→					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   194→					return nil
   195→				}
   196→
   197→				walPath := path.Join(hostDir, "current.wal")
   198→				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   199→				if err != nil {
   200→					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)


Found 3 files
pkg/metricstore/walCheckpoint.go
pkg/metricstore/metricstore.go
pkg/metricstore/lineprotocol.go


   200→					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   201→					return nil
   202→				}
   203→
   204→				w := bufio.NewWriter(f)
   205→
   206→				// Write file header magic if file is new (empty).
   207→				info, err := f.Stat()
   208→				if err == nil && info.Size() == 0 {
   209→					var hdr [4]byte
   210→					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   211→					if _, err := w.Write(hdr[:]); err != nil {
   212→						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   213→						f.Close()
   214→						return nil
   215→					}
   216→				}
   217→
   218→				ws = &walFileState{f: f, w: w}
   219→				hostFiles[hostDir] = ws
   220→				return ws
   221→			}
   222→
   223→			processMsg := func(msg *WALMessage) {
   224→				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   225→				ws := getOrOpenWAL(hostDir)
   226→				if ws == nil {
   227→					return
   228→				}
   229→				if err := writeWALRecordDirect(ws.w, msg); err != nil {
   230→					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   231→				}
   232→				ws.dirty = true
   233→			}
   234→
   235→			processRotate := func(req walRotateReq) {
   236→				ws, ok := hostFiles[req.hostDir]
   237→				if ok && ws.f != nil {
   238→					ws.w.Flush()
   239→					ws.f.Close()
   240→					walPath := path.Join(req.hostDir, "current.wal")
   241→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   242→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   243→					}
   244→					delete(hostFiles, req.hostDir)
   245→				}
   246→				close(req.done)
   247→			}
   248→
   249→			flushDirty := func() {
   250→				for _, ws := range hostFiles {
   251→					if ws.dirty {
   252→						ws.w.Flush()
   253→						ws.dirty = false
   254→					}
   255→				}
   256→			}
   257→
   258→			drain := func() {
   259→				for {
   260→					select {
   261→					case msg, ok := <-msgCh:
   262→						if !ok {
   263→							return
   264→						}
   265→						processMsg(msg)
   266→					case req := <-rotateCh:
   267→						processRotate(req)
   268→					default:
   269→						flushDirty()
   270→						return
   271→					}
   272→				}
   273→			}
   274→
   275→			ticker := time.NewTicker(walFlushInterval)
   276→			defer ticker.Stop()
   277→
   278→			// drainBatch processes up to 4096 pending messages without blocking.
   279→			// Returns false if the channel was closed.
   280→			drainBatch := func() bool {
   281→				for range 4096 {
   282→					select {
   283→					case msg, ok := <-msgCh:
   284→						if !ok {
   285→							flushDirty()
   286→							return false
   287→						}
   288→						processMsg(msg)
   289→					case req := <-rotateCh:
   290→						processRotate(req)
   291→					default:
   292→						return true
   293→					}
   294→				}
   295→				return true
   296→			}
   297→
   298→			for {
   299→				select {
   300→				case <-ctx.Done():
   301→					drain()
   302→					return
   303→				case msg, ok := <-msgCh:
   304→					if !ok {
   305→						return
   306→					}
   307→					processMsg(msg)
   308→					if !drainBatch() {
   309→						return
   310→					}
   311→					// No flush here — timer handles periodic flushing.
   312→				case <-ticker.C:
   313→					flushDirty()
   314→				case req := <-rotateCh:
   315→					processRotate(req)
   316→				}
   317→			}
   318→		})
   319→	}
   320→}
   321→
   322→// RotateWALFiles sends rotation requests for the given host directories
   323→// and blocks until all rotations complete. Each request is routed to the
   324→// shard that owns the host directory.
   325→func RotateWALFiles(hostDirs []string) {
   326→	if walShardRotateChs == nil {
   327→		return
   328→	}
   329→	dones := make([]chan struct{}, len(hostDirs))
   330→	for i, dir := range hostDirs {
   331→		dones[i] = make(chan struct{})
   332→		// Extract cluster/node from hostDir to find the right shard.
   333→		// hostDir = rootDir/cluster/node
   334→		shard := walShardIndexFromDir(dir)
   335→		walShardRotateChs[shard] <- walRotateReq{hostDir: dir, done: dones[i]}
   336→	}
   337→	for _, done := range dones {
   338→		<-done
   339→	}
   340→}
   341→
   342→// walShardIndexFromDir extracts cluster and node from a host directory path
   343→// and returns the shard index. The path format is: rootDir/cluster/node
   344→func walShardIndexFromDir(hostDir string) int {
   345→	// hostDir = .../cluster/node — extract last two path components
   346→	node := path.Base(hostDir)
   347→	cluster := path.Base(path.Dir(hostDir))
   348→	return walShardIndex(cluster, node)
   349→}
   350→
   351→// RotateWALFiles sends rotation requests for the given host directories
   352→// and blocks until all rotations complete.
   353→func RotateWALFilesAfterShutdown(hostDirs []string) {
   354→	for _, dir := range hostDirs {
   355→		walPath := path.Join(dir, "current.wal")
   356→		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   357→			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   358→		}
   359→	}
   360→}
   361→
   362→// writeWALRecordDirect encodes a WAL record directly into the bufio.Writer,
   363→// avoiding heap allocations by using a stack-allocated scratch buffer for
   364→// the fixed-size header/trailer and computing CRC inline.
   365→func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
   366→	// Compute payload size.
   367→	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
   368→	for _, s := range msg.Selector {
   369→		payloadSize += 1 + len(s)
   370→	}
   371→
   372→	// Write magic + payload length (8 bytes header).
   373→	var hdr [8]byte
   374→	binary.LittleEndian.PutUint32(hdr[0:4], walRecordMagic)
   375→	binary.LittleEndian.PutUint32(hdr[4:8], uint32(payloadSize))
   376→	if _, err := w.Write(hdr[:]); err != nil {
   377→		return err
   378→	}
   379→
   380→	// We need to compute CRC over the payload as we write it.
   381→	crc := crc32.NewIEEE()
   382→
   383→	// Timestamp (8 bytes).
   384→	var scratch [8]byte
   385→	binary.LittleEndian.PutUint64(scratch[:8], uint64(msg.Timestamp))
   386→	crc.Write(scratch[:8])
   387→	if _, err := w.Write(scratch[:8]); err != nil {
   388→		return err
   389→	}
   390→
   391→	// Metric name length (2 bytes) + metric name.
   392→	binary.LittleEndian.PutUint16(scratch[:2], uint16(len(msg.MetricName)))
   393→	crc.Write(scratch[:2])
   394→	if _, err := w.Write(scratch[:2]); err != nil {
   395→		return err
   396→	}
   397→	nameBytes := []byte(msg.MetricName)
   398→	crc.Write(nameBytes)
   399→	if _, err := w.Write(nameBytes); err != nil {
   400→		return err
   401→	}
   402→
   403→	// Selector count (1 byte).
   404→	scratch[0] = byte(len(msg.Selector))
   405→	crc.Write(scratch[:1])
   406→	if _, err := w.Write(scratch[:1]); err != nil {
   407→		return err
   408→	}
   409→
   410→	// Selectors (1-byte length + bytes each).
   411→	for _, sel := range msg.Selector {
   412→		scratch[0] = byte(len(sel))
   413→		crc.Write(scratch[:1])
   414→		if _, err := w.Write(scratch[:1]); err != nil {
   415→			return err
   416→		}
   417→		selBytes := []byte(sel)
   418→		crc.Write(selBytes)
   419→		if _, err := w.Write(selBytes); err != nil {
   420→			return err
   421→		}
   422→	}
   423→
   424→	// Value (4 bytes, float32 bits).
   425→	binary.LittleEndian.PutUint32(scratch[:4], math.Float32bits(float32(msg.Value)))
   426→	crc.Write(scratch[:4])
   427→	if _, err := w.Write(scratch[:4]); err != nil {
   428→		return err
   429→	}
   430→
   431→	// CRC32 (4 bytes).
   432→	binary.LittleEndian.PutUint32(scratch[:4], crc.Sum32())
   433→	_, err := w.Write(scratch[:4])
   434→	return err
   435→}
   436→
   437→// readWALRecord reads one WAL record from the reader.
   438→// Returns (nil, nil) on clean EOF. Returns error on data corruption.
   439→// A CRC mismatch indicates a truncated trailing record (expected on crash).
   440→func readWALRecord(r io.Reader) (*WALMessage, error) {
   441→	var magic uint32
   442→	if err := binary.Read(r, binary.LittleEndian, &magic); err != nil {
   443→		if err == io.EOF {
   444→			return nil, nil // Clean EOF
   445→		}
   446→		return nil, fmt.Errorf("read record magic: %w", err)
   447→	}
   448→
   449→	if magic != walRecordMagic {


889b284f Checkpoint: ed68d32218ac
6f7dda53 Cleanup
bc9c47cd Checkpoint: ddd40d290c56
0325d9e8 fix: Increase throughput for WAL writers
c13fd68a Merge pull request #534 from ClusterCockpit/hotfix
3d94b0bf Merge branch 'master' into hotfix
d5ea2b4c change: query node states explicitly in node view
7c42fc3a Checkpoint: 29b9d52db89c
45f329e5 feat: Add command line switch to trigger manual metricstore checkpoint cleanup
100dd7da Merge pull request #533 from ClusterCockpit/hotfix
493cee05 Checkpoint: ccda3b2ff4cb
192c94a7 fix: Prevent interruption of body lineprotocol parsing on locks
40142966 Checkpoint: 6000eb5a5bb8
e41d1251 fix: Continue on error
1abea621 Checkpoint: 1660b8cf2571
586c9020 Restructure metricstore cleanup archiving to stay withinh 32k parquet-go limit
38f75ceb Checkpoint: 1660b8cf2571
7297f7f3 Merge remote session logs
01ec70ba Iterate over subCluster MetricConfig directly so that removed metrics are not included
92189ea9 Checkpoint: efb6f0a96069


commit 0325d9e866ec97994e77770ca189838c0ee3b3f0
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Tue Mar 24 06:53:12 2026 +0100

    fix: Increase throughput for WAL writers

    Entire-Checkpoint: ddd40d290c56

 pkg/metricstore/walCheckpoint.go | 63 ++++++++++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 21 deletions(-)


commit 0325d9e866ec97994e77770ca189838c0ee3b3f0
Author: Jan Eitzinger <jan@moebiusband.org>
Date:   Tue Mar 24 06:53:12 2026 +0100

    fix: Increase throughput for WAL writers

    Entire-Checkpoint: ddd40d290c56

diff --git a/pkg/metricstore/walCheckpoint.go b/pkg/metricstore/walCheckpoint.go
index 38585cf5..85e5047e 100644
--- a/pkg/metricstore/walCheckpoint.go
+++ b/pkg/metricstore/walCheckpoint.go
@@ -69,6 +69,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"time"

 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
 	"github.com/ClusterCockpit/cc-lib/v2/schema"
@@ -111,10 +112,16 @@ type walRotateReq struct {

 // walFileState holds an open WAL file handle and buffered writer for one host directory.
 type walFileState struct {
-	f *os.File
-	w *bufio.Writer
+	f     *os.File
+	w     *bufio.Writer
+	dirty bool
 }

+// walFlushInterval controls how often dirty WAL files are flushed to disk.
+// Decoupling flushes from message processing lets the consumer run at memory
+// speed, amortizing syscall overhead across many writes.
+const walFlushInterval = 5 * time.Second
+
 // walShardIndex computes which shard a message belongs to based on cluster+node.
 // Uses FNV-1a hash for fast, well-distributed mapping.
 func walShardIndex(cluster, node string) int {
@@ -222,6 +229,7 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 				if err := writeWALRecordDirect(ws.w, msg); err != nil {
 					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
 				}
+				ws.dirty = true
 			}

 			processRotate := func(req walRotateReq) {
@@ -238,10 +246,11 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 				close(req.done)
 			}

-			flushAll := func() {
+			flushDirty := func() {
 				for _, ws := range hostFiles {
-					if ws.f != nil {
+					if ws.dirty {
 						ws.w.Flush()
+						ws.dirty = false
 					}
 				}
 			}
@@ -257,12 +266,35 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 					case req := <-rotateCh:
 						processRotate(req)
 					default:
-						flushAll()
+						flushDirty()
 						return
 					}
 				}
 			}

+			ticker := time.NewTicker(walFlushInterval)
+			defer ticker.Stop()
+
+			// drainBatch processes up to 4096 pending messages without blocking.
+			// Returns false if the channel was closed.
+			drainBatch := func() bool {
+				for range 4096 {
+					select {
+					case msg, ok := <-msgCh:
+						if !ok {
+							flushDirty()
+							return false
+						}
+						processMsg(msg)
+					case req := <-rotateCh:
+						processRotate(req)
+					default:
+						return true
+					}
+				}
+				return true
+			}
+
 			for {
 				select {
 				case <-ctx.Done():
@@ -273,23 +305,12 @@ func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
 						return
 					}
 					processMsg(msg)
-
-					// Drain up to 256 more messages without blocking to batch writes.
-					for range 256 {
-						select {
-						case msg, ok := <-msgCh:
-							if !ok {
-								return
-							}
-							processMsg(msg)
-						case req := <-rotateCh:
-							processRotate(req)
-						default:
-							goto flushed
-						}
+					if !drainBatch() {
+						return
 					}
-				flushed:
-					flushAll()
+					// No flush here — timer handles periodic flushing.
+				case <-ticker.C:
+					flushDirty()
 				case req := <-rotateCh:
 					processRotate(req)
 				}


Found 6 files
internal/api/api_test.go
internal/archiver/archiver.go
cmd/cc-backend/server.go
pkg/metricstore/checkpoint.go
internal/archiver/archiveWorker.go
internal/api/nats_test.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements checkpoint persistence for the in-memory metric store.
     7→//
     8→// Checkpoints enable graceful restarts by periodically saving in-memory metric
     9→// data to disk in JSON or binary format. The checkpoint system:
    10→//
    11→// Key Features:
    12→//   - Periodic background checkpointing via the Checkpointing() worker
    13→//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
    14→//   - Parallel checkpoint creation and loading using worker pools
    15→//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
    16→//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
    17→//   - Only saves unarchived data (archived data is already persisted elsewhere)
    18→//   - GC optimization during loading to prevent excessive heap growth
    19→//
    20→// Checkpoint Workflow:
    21→//  1. Init() loads checkpoints within retention window at startup
    22→//  2. Checkpointing() worker periodically saves new data
    23→//  3. Shutdown() writes final checkpoint before exit
    24→//
    25→// File Organization:
    26→//
    27→//	checkpoints/
    28→//	  cluster1/
    29→//	    host001/
    30→//	      1234567890.json  (JSON format: full subtree snapshot)
    31→//	      1234567890.bin   (binary format: full subtree snapshot)
    32→//	      current.wal      (WAL format: append-only per-entry log)
    33→//	    host002/
    34→//	      ...
    35→package metricstore
    36→
    37→import (
    38→	"bufio"
    39→	"context"
    40→	"encoding/json"
    41→	"errors"
    42→	"fmt"
    43→	"io/fs"
    44→	"os"
    45→	"path"
    46→	"path/filepath"
    47→	"sort"
    48→	"strconv"
    49→	"strings"
    50→	"sync"
    51→	"sync/atomic"
    52→	"time"
    53→
    54→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    55→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    56→)
    57→
    58→const (
    59→	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
    60→	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
    61→	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
    62→)
    63→
    64→// CheckpointMetrics represents metric data in a checkpoint file.
    65→// Whenever the structure changes, update MarshalJSON as well!
    66→type CheckpointMetrics struct {
    67→	Data      []schema.Float `json:"data"`
    68→	Frequency int64          `json:"frequency"`
    69→	Start     int64          `json:"start"`
    70→}
    71→
    72→// CheckpointFile represents the hierarchical structure of a checkpoint file.
    73→// It mirrors the Level tree structure from the MemoryStore.
    74→type CheckpointFile struct {
    75→	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
    76→	Children map[string]*CheckpointFile    `json:"children"`
    77→	From     int64                         `json:"from"`
    78→	To       int64                         `json:"to"`
    79→}
    80→
    81→// lastCheckpoint tracks the timestamp of the last checkpoint creation.
    82→var (
    83→	lastCheckpoint   time.Time
    84→	lastCheckpointMu sync.Mutex
    85→)
    86→
    87→// Checkpointing starts a background worker that periodically saves metric data to disk.
    88→//
    89→// Checkpoints are written every 12 hours (hardcoded).
    90→//
    91→// Format behaviour:
    92→//   - "json": Periodic checkpointing every checkpointInterval
    93→//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
    94→func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
    95→	lastCheckpointMu.Lock()
    96→	lastCheckpoint = time.Now()
    97→	lastCheckpointMu.Unlock()
    98→
    99→	ms := GetMemoryStore()
   100→
   101→	wg.Go(func() {
   102→
   103→		d := 12 * time.Hour // default checkpoint interval
   104→		if Keys.CheckpointInterval != "" {
   105→			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
   106→			if err != nil {
   107→				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
   108→			} else {
   109→				d = parsed
   110→			}
   111→		}
   112→
   113→		ticker := time.NewTicker(d)
   114→		defer ticker.Stop()
   115→
   116→		for {
   117→			select {
   118→			case <-ctx.Done():
   119→				return
   120→			case <-ticker.C:
   121→				lastCheckpointMu.Lock()
   122→				from := lastCheckpoint
   123→				lastCheckpointMu.Unlock()
   124→
   125→				now := time.Now()
   126→				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
   127→
   128→				if Keys.Checkpoints.FileFormat == "wal" {
   129→					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   130→					if err != nil {
   131→						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
   132→					} else {
   133→						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
   134→						lastCheckpointMu.Lock()
   135→						lastCheckpoint = now
   136→						lastCheckpointMu.Unlock()
   137→						// Rotate WAL files for successfully checkpointed hosts.
   138→						RotateWALFiles(hostDirs)
   139→					}
   140→				} else {
   141→					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   142→					if err != nil {
   143→						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
   144→					} else {
   145→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
   146→						lastCheckpointMu.Lock()
   147→						lastCheckpoint = now
   148→						lastCheckpointMu.Unlock()
   149→					}
   150→				}
   151→			}
   152→		}
   153→	})
   154→}
   155→
   156→// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
   157→//
   158→// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
   159→// This method manually constructs JSON to avoid allocations and interface conversions.
   160→func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
   161→	buf := make([]byte, 0, 128+len(cm.Data)*8)
   162→	buf = append(buf, `{"frequency":`...)
   163→	buf = strconv.AppendInt(buf, cm.Frequency, 10)
   164→	buf = append(buf, `,"start":`...)
   165→	buf = strconv.AppendInt(buf, cm.Start, 10)
   166→	buf = append(buf, `,"data":[`...)
   167→	for i, x := range cm.Data {
   168→		if i != 0 {
   169→			buf = append(buf, ',')
   170→		}
   171→		if x.IsNaN() {
   172→			buf = append(buf, `null`...)
   173→		} else {
   174→			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
   175→		}
   176→	}
   177→	buf = append(buf, `]}`...)
   178→	return buf, nil
   179→}
   180→
   181→// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format).
   182→//
   183→// Metrics at root and cluster levels are skipped. One file per host is created.
   184→// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
   185→// at a time, allowing concurrent writes/reads to other hosts.
   186→//
   187→// Returns the number of checkpoint files created and any errors encountered.
   188→func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
   189→	// Pre-calculate capacity by counting cluster/host pairs
   190→	m.root.lock.RLock()
   191→	totalHosts := 0
   192→	for _, l1 := range m.root.children {
   193→		l1.lock.RLock()
   194→		totalHosts += len(l1.children)
   195→		l1.lock.RUnlock()
   196→	}
   197→	m.root.lock.RUnlock()
   198→
   199→	levels := make([]*Level, 0, totalHosts)
   200→	selectors := make([][]string, 0, totalHosts)


Bash

grep -n "RotateWALFiles\|drain\|select.*ctx.Done" /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go | head -40


258:			drain := func() {
278:			// drainBatch processes up to 4096 pending messages without blocking.
280:			drainBatch := func() bool {
301:					drain()
308:					if !drainBatch() {
322:// RotateWALFiles sends rotation requests for the given host directories
325:func RotateWALFiles(hostDirs []string) {
351:// RotateWALFiles sends rotation requests for the given host directories
353:func RotateWALFilesAfterShutdown(hostDirs []string) {


   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→
   290→	ms := GetMemoryStore()
   291→
   292→	lastCheckpointMu.Lock()
   293→	from := lastCheckpoint
   294→	lastCheckpointMu.Unlock()
   295→
   296→	if Keys.Checkpoints.FileFormat == "wal" {
   297→		var hostDirs []string
   298→		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   299→		if err == nil {
   300→			RotateWALFilesAfterShutdown(hostDirs)
   301→		}
   302→	} else {
   303→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   304→	}
   305→
   306→	if err != nil {
   307→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
   308→	}
   309→	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
   310→}
   311→
   312→// Retention starts a background goroutine that periodically frees old metric data.
   313→//
   314→// This worker runs at half the retention interval and calls Free() to remove buffers
   315→// older than the configured retention time. It respects the NodeProvider to preserve
   316→// data for nodes with active jobs.
   317→//
   318→// Parameters:
   319→//   - wg: WaitGroup to signal completion when context is cancelled
   320→//   - ctx: Context for cancellation signal
   321→//
   322→// The goroutine exits when ctx is cancelled.


grep -n "RotateWALFiles\|ToCheckpointWAL\|close(ch)" /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go | head -20


282:			close(ch)
298:		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
300:			RotateWALFilesAfterShutdown(hostDirs)


grep -n "func (m \*MemoryStore) ToCheckpointWAL\|func.*ToCheckpoint" /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/*.go


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go:188:func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go:627:func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {


   627→func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
   628→	// Collect all cluster/host pairs.
   629→	m.root.lock.RLock()
   630→	totalHosts := 0
   631→	for _, l1 := range m.root.children {
   632→		l1.lock.RLock()
   633→		totalHosts += len(l1.children)
   634→		l1.lock.RUnlock()
   635→	}
   636→	m.root.lock.RUnlock()
   637→
   638→	levels := make([]*Level, 0, totalHosts)
   639→	selectors := make([][]string, 0, totalHosts)
   640→
   641→	m.root.lock.RLock()
   642→	for sel1, l1 := range m.root.children {
   643→		l1.lock.RLock()
   644→		for sel2, l2 := range l1.children {
   645→			levels = append(levels, l2)
   646→			selectors = append(selectors, []string{sel1, sel2})
   647→		}
   648→		l1.lock.RUnlock()
   649→	}
   650→	m.root.lock.RUnlock()
   651→
   652→	type workItem struct {
   653→		level    *Level
   654→		hostDir  string
   655→		selector []string
   656→	}
   657→
   658→	n, errs := int32(0), int32(0)
   659→	var successDirs []string
   660→	var successMu sync.Mutex
   661→
   662→	var wg sync.WaitGroup
   663→	wg.Add(Keys.NumWorkers)
   664→	work := make(chan workItem, Keys.NumWorkers*2)
   665→
   666→	for range Keys.NumWorkers {
   667→		go func() {
   668→			defer wg.Done()
   669→			for wi := range work {
   670→				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
   671→				if err != nil {
   672→					if err == ErrNoNewArchiveData {
   673→						continue
   674→					}
   675→					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
   676→					atomic.AddInt32(&errs, 1)
   677→				} else {
   678→					atomic.AddInt32(&n, 1)
   679→					successMu.Lock()
   680→					successDirs = append(successDirs, wi.hostDir)
   681→					successMu.Unlock()
   682→				}
   683→			}
   684→		}()
   685→	}
   686→
   687→	for i := range levels {
   688→		hostDir := path.Join(dir, path.Join(selectors[i]...))
   689→		work <- workItem{
   690→			level:    levels[i],
   691→			hostDir:  hostDir,
   692→			selector: selectors[i],
   693→		}
   694→	}
   695→	close(work)
   696→	wg.Wait()
   697→
   698→	if errs > 0 {
   699→		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
   700→	}
   701→	return int(n), successDirs, nil
   702→}
   703→
   704→// toCheckpointBinary writes a binary snapshot file for a single host-level node.
   705→// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
   706→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   707→	cf, err := l.toCheckpointFile(from, to, m)
   708→	if err != nil {
   709→		return err
   710→	}
   711→	if cf == nil {
   712→		return ErrNoNewArchiveData
   713→	}
   714→
   715→	if err := os.MkdirAll(dir, CheckpointDirPerms); err != nil {
   716→		return fmt.Errorf("mkdir %s: %w", dir, err)
   717→	}
   718→
   719→	// Write to a temp file first, then rename (atomic on POSIX).
   720→	tmpPath := path.Join(dir, fmt.Sprintf("%d.bin.tmp", from))
   721→	finalPath := path.Join(dir, fmt.Sprintf("%d.bin", from))
   722→
   723→	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   724→	if err != nil {
   725→		return fmt.Errorf("open binary snapshot %s: %w", tmpPath, err)
   726→	}
   727→
   728→	bw := bufio.NewWriter(f)
   729→	if err := writeBinarySnapshotFile(bw, cf); err != nil {
   730→		f.Close()
   731→		os.Remove(tmpPath)
   732→		return fmt.Errorf("write binary snapshot: %w", err)
   733→	}
   734→	if err := bw.Flush(); err != nil {
   735→		f.Close()
   736→		os.Remove(tmpPath)
   737→		return err
   738→	}
   739→	f.Close()
   740→
   741→	return os.Rename(tmpPath, finalPath)
   742→}
   743→
   744→// writeBinarySnapshotFile writes the binary snapshot file header and level tree.
   745→func writeBinarySnapshotFile(w io.Writer, cf *CheckpointFile) error {
   746→	if err := binary.Write(w, binary.LittleEndian, snapFileMagic); err != nil {
   747→		return err
   748→	}
   749→	if err := binary.Write(w, binary.LittleEndian, cf.From); err != nil {
   750→		return err
   751→	}
   752→	if err := binary.Write(w, binary.LittleEndian, cf.To); err != nil {
   753→		return err
   754→	}
   755→	return writeBinaryLevel(w, cf)
   756→}
   757→
   758→// writeBinaryLevel recursively writes a CheckpointFile level in binary format.
   759→func writeBinaryLevel(w io.Writer, cf *CheckpointFile) error {
   760→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Metrics))); err != nil {
   761→		return err
   762→	}
   763→
   764→	for name, metric := range cf.Metrics {
   765→		if err := writeString16(w, name); err != nil {
   766→			return err
   767→		}
   768→		if err := binary.Write(w, binary.LittleEndian, metric.Frequency); err != nil {
   769→			return err
   770→		}
   771→		if err := binary.Write(w, binary.LittleEndian, metric.Start); err != nil {
   772→			return err
   773→		}
   774→		if err := binary.Write(w, binary.LittleEndian, uint32(len(metric.Data))); err != nil {
   775→			return err
   776→		}
   777→		for _, v := range metric.Data {
   778→			if err := binary.Write(w, binary.LittleEndian, math.Float32bits(float32(v))); err != nil {
   779→				return err
   780→			}
   781→		}
   782→	}
   783→
   784→	if err := binary.Write(w, binary.LittleEndian, uint32(len(cf.Children))); err != nil {
   785→		return err
   786→	}
   787→
   788→	for name, child := range cf.Children {
   789→		if err := writeString16(w, name); err != nil {
   790→			return err
   791→		}
   792→		if err := writeBinaryLevel(w, child); err != nil {
   793→			return err
   794→		}
   795→	}
   796→
   797→	return nil
   798→}
   799→
   800→// writeString16 writes a 2-byte length-prefixed string to w.
   801→func writeString16(w io.Writer, s string) error {
   802→	if err := binary.Write(w, binary.LittleEndian, uint16(len(s))); err != nil {
   803→		return err
   804→	}
   805→	_, err := io.WriteString(w, s)
   806→	return err
   807→}
   808→
   809→// loadBinaryFile reads a binary snapshot file and loads data into the Level tree.
   810→// The retention check (from) is applied to the file's 'to' timestamp.
   811→func (l *Level) loadBinaryFile(m *MemoryStore, f *os.File, from int64) error {
   812→	br := bufio.NewReader(f)
   813→
   814→	var magic uint32
   815→	if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
   816→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read magic: %w", err)
   817→	}
   818→	if magic != snapFileMagic {
   819→		return fmt.Errorf("[METRICSTORE]> binary snapshot: invalid magic 0x%08X (expected 0x%08X)", magic, snapFileMagic)
   820→	}
   821→
   822→	var fileFrom, fileTo int64
   823→	if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
   824→		return fmt.Errorf("[METRICSTORE]> binary snapshot: read from: %w", err)
   825→	}
   826→	if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {


   400→	}
   401→
   402→	// Handle context cancellation for graceful shutdown
   403→	go func() {
   404→		<-ctx.Done()
   405→		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
   406→		defer cancel()
   407→		if err := s.server.Shutdown(shutdownCtx); err != nil {
   408→			cclog.Errorf("Server shutdown error: %v", err)
   409→		}
   410→	}()
   411→
   412→	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
   413→		return fmt.Errorf("server failed: %w", err)
   414→	}
   415→	return nil
   416→}
   417→
   418→func (s *Server) Shutdown(ctx context.Context) {
   419→	// Create a shutdown context with timeout
   420→	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
   421→	defer cancel()
   422→
   423→	nc := nats.GetClient()
   424→	if nc != nil {
   425→		nc.Close()
   426→	}
   427→
   428→	// First shut down the server gracefully (waiting for all ongoing requests)
   429→	if err := s.server.Shutdown(shutdownCtx); err != nil {
   430→		cclog.Errorf("Server shutdown error: %v", err)
   431→	}
   432→
   433→	// Archive all the metric store data
   434→	ms := metricstore.GetMemoryStore()
   435→
   436→	if ms != nil {
   437→		metricstore.Shutdown()
   438→	}
   439→
   440→	// Shutdown archiver with 10 second timeout for fast shutdown
   441→	if err := archiver.Shutdown(10 * time.Second); err != nil {
   442→		cclog.Warnf("Archiver shutdown: %v", err)
   443→	}
   444→}
   445→


   418→func (s *Server) Shutdown(ctx context.Context) {
   419→	// Create a shutdown context with timeout
   420→	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
   421→	defer cancel()
   422→
   423→	nc := nats.GetClient()
   424→	if nc != nil {
   425→		nc.Close()
   426→	}
   427→
   428→	// First shut down the server gracefully (waiting for all ongoing requests)
   429→	if err := s.server.Shutdown(shutdownCtx); err != nil {
   430→		cclog.Errorf("Server shutdown error: %v", err)
   431→	}
   432→
   433→	// Archive all the metric store data
   434→	ms := metricstore.GetMemoryStore()
   435→
   436→	if ms != nil {
   437→		metricstore.Shutdown()
   438→	}
   439→
   440→	// Shutdown archiver with 10 second timeout for fast shutdown
   441→	if err := archiver.Shutdown(10 * time.Second); err != nil {
   442→		cclog.Warnf("Archiver shutdown: %v", err)
   443→	}
   444→}
   445→


   339→func runServer(ctx context.Context) error {
   340→	var wg sync.WaitGroup
   341→
   342→	// Initialize metric store if configuration is provided
   343→	haveMetricstore := false
   344→	mscfg := ccconf.GetPackageConfig("metric-store")
   345→	if mscfg != nil {
   346→		metrics := metricstore.BuildMetricList()
   347→		metricstore.Init(mscfg, metrics, &wg)
   348→
   349→		// Inject repository as NodeProvider to break import cycle
   350→		ms := metricstore.GetMemoryStore()
   351→		jobRepo := repository.GetJobRepository()
   352→		ms.SetNodeProvider(jobRepo)
   353→		metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
   354→		haveMetricstore = true
   355→	} else {
   356→		metricstore.MetricStoreHandle = nil
   357→		cclog.Debug("missing internal metricstore configuration")
   358→	}
   359→
   360→	// Initialize external metric stores if configuration is provided
   361→	mscfg = ccconf.GetPackageConfig("metric-store-external")
   362→	if mscfg != nil {
   363→		err := metricdispatch.Init(mscfg)
   364→
   365→		if err != nil {
   366→			cclog.Debugf("error while initializing external metricdispatch: %v", err)
   367→		} else {
   368→			haveMetricstore = true
   369→		}
   370→	}
   371→
   372→	if !haveMetricstore {
   373→		return fmt.Errorf("missing metricstore configuration")
   374→	}
   375→
   376→	// Start archiver and task manager
   377→	archiver.Start(repository.GetJobRepository(), ctx)
   378→	taskmanager.Start(ccconf.GetPackageConfig("cron"), ccconf.GetPackageConfig("archive"))
   379→
   380→	// Initialize web UI
   381→	cfg := ccconf.GetPackageConfig("ui")
   382→	if err := web.Init(cfg); err != nil {
   383→		return fmt.Errorf("initializing web UI: %w", err)
   384→	}
   385→
   386→	// Initialize HTTP server
   387→	srv, err := NewServer(version, commit, date)
   388→	if err != nil {
   389→		return fmt.Errorf("creating server: %w", err)
   390→	}
   391→
   392→	// Channel to collect errors from server
   393→	errChan := make(chan error, 1)
   394→
   395→	// Start HTTP server
   396→	wg.Go(func() {
   397→		if err := srv.Start(ctx); err != nil {
   398→			errChan <- err
   399→		}
   400→	})
   401→
   402→	// Handle shutdown signals
   403→	wg.Add(1)
   404→	sigs := make(chan os.Signal, 1)
   405→	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
   406→	go func() {
   407→		defer wg.Done()
   408→		select {
   409→		case <-sigs:
   410→			cclog.Info("Shutdown signal received")
   411→		case <-ctx.Done():
   412→		}
   413→
   414→		runtime.SystemdNotify(false, "Shutting down ...")
   415→		srv.Shutdown(ctx)
   416→		util.FsWatcherShutdown()
   417→		taskmanager.Shutdown()
   418→	}()
   419→
   420→	// Set GC percent if not configured
   421→	if os.Getenv(envGOGC) == "" {
   422→		// trigger GC when heap grows 15% above the previous live set
   423→		debug.SetGCPercent(15)
   424→	}
   425→	runtime.SystemdNotify(true, "running")
   426→
   427→	waitDone := make(chan struct{})
   428→	go func() {
   429→		wg.Wait()
   430→		close(waitDone)
   431→	}()
   432→
   433→	go func() {
   434→		<-waitDone
   435→		close(errChan)
   436→	}()
   437→
   438→	// Wait for either:
   439→	// 1. An error from server startup
   440→	// 2. Completion of all goroutines (normal shutdown or crash)
   441→	select {
   442→	case err := <-errChan:
   443→		// errChan will be closed when waitDone is closed, which happens
   444→		// when all goroutines complete (either from normal shutdown or error)
   445→		if err != nil {
   446→			return err
   447→		}
   448→	case <-time.After(100 * time.Millisecond):
   449→		// Give the server 100ms to start and report any immediate startup errors
   450→		// After that, just wait for normal shutdown completion
   451→		select {
   452→		case err := <-errChan:
   453→			if err != nil {
   454→				return err
   455→			}
   456→		case <-waitDone:
   457→			// Normal shutdown completed
   458→		}
   459→	}
   460→
   461→	cclog.Print("Graceful shutdown completed!")
   462→	return nil
   463→}
   464→
   465→func run() error {
   466→	cliInit()
   467→
   468→	if flagVersion {


   254→					}
   255→				}
   256→			}
   257→
   258→			drain := func() {
   259→				for {
   260→					select {
   261→					case msg, ok := <-msgCh:
   262→						if !ok {
   263→							return
   264→						}
   265→						processMsg(msg)
   266→					case req := <-rotateCh:
   267→						processRotate(req)
   268→					default:
   269→						flushDirty()
   270→						return
   271→					}
   272→				}
   273→			}
   274→
   275→			ticker := time.NewTicker(walFlushInterval)
   276→			defer ticker.Stop()
   277→
   278→			// drainBatch processes up to 4096 pending messages without blocking.
   279→			// Returns false if the channel was closed.
   280→			drainBatch := func() bool {
   281→				for range 4096 {
   282→					select {
   283→					case msg, ok := <-msgCh:
   284→						if !ok {
   285→							flushDirty()
   286→							return false
   287→						}
   288→						processMsg(msg)
   289→					case req := <-rotateCh:
   290→						processRotate(req)
   291→					default:
   292→						return true
   293→					}
   294→				}
   295→				return true
   296→			}
   297→
   298→			for {
   299→				select {
   300→				case <-ctx.Done():
   301→					drain()
   302→					return
   303→				case msg, ok := <-msgCh:
   304→					if !ok {
   305→						return
   306→					}
   307→					processMsg(msg)
   308→					if !drainBatch() {
   309→						return
   310→					}
   311→					// No flush here — timer handles periodic flushing.
   312→				case <-ticker.C:
   313→					flushDirty()
   314→				case req := <-rotateCh:
   315→					processRotate(req)
   316→				}
   317→			}
   318→		})
   319→	}
   320→}
   321→
   322→// RotateWALFiles sends rotation requests for the given host directories
   323→// and blocks until all rotations complete. Each request is routed to the
   324→// shard that owns the host directory.
   325→func RotateWALFiles(hostDirs []string) {
   326→	if walShardRotateChs == nil {
   327→		return
   328→	}


Perfect! Now I have all the information I need. Let me create a comprehensive report:

## Summary: Shutdown Flow and Worker Pool Analysis

I've completed a thorough analysis of the shutdown flow and worker pool implementations. The recent increase in shutdown time from ~10s to ~40s is likely caused by a recent change to the WAL staging worker pool combined with how shutdown orchestration works. Here are my findings:

### 1. **MAIN SHUTDOWN FLOW** (`cmd/cc-backend/main.go`)

**Signal Handling (lines 402-418):**
- Receives SIGINT/SIGTERM on channel (line 405)
- Calls `srv.Shutdown(ctx)` (line 415)
- Calls `util.FsWatcherShutdown()` (line 416)
- Calls `taskmanager.Shutdown()` (line 417)
- Waits for all goroutines to complete via `wg.Wait()` (lines 428-431)

**Key Timeouts:**
- HTTP server shutdown: **30 seconds** (line 420 in `server.go`)
- Archiver shutdown: **10 seconds** (line 441 in `server.go`)
- NO TIMEOUT on metricstore shutdown (line 437 in `server.go` - **THIS IS BLOCKING**)

---

### 2. **SERVER SHUTDOWN** (`cmd/cc-backend/server.go:418-444`)

**Current Implementation:**
```go
func (s *Server) Shutdown(ctx context.Context) {
    shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
    defer cancel()

    // 1. Close NATS client (line 425)
    // 2. HTTP server graceful shutdown with 30s timeout (lines 429-431)
    // 3. metricstore.Shutdown() - NO TIMEOUT (line 437) ⚠
    // 4. archiver.Shutdown(10s) (line 441)
}
```

**Problem Areas:**
- **metricstore.Shutdown()** (line 437) has **NO TIMEOUT** and performs blocking I/O operations
- This function writes final checkpoint files synchronously, which can take significant time

---

### 3. **METRICSTORE SHUTDOWN** (`pkg/metricstore/metricstore.go:273-310`)

**Key Steps:**
1. Cancels context to stop all background workers (lines 276-278)
2. **Closes WAL shard channels** (lines 280-283) - triggers shutdown of all WAL staging goroutines
3. **Writes final checkpoint** (lines 296-304) - **THIS IS BLOCKING AND CAN BE SLOW**
   - Calls `ms.ToCheckpointWAL()` for binary format (line 298)
   - Calls `RotateWALFilesAfterShutdown()` (line 300)

**The WAL Checkpoint Problem:**
- `ToCheckpointWAL()` (walCheckpoint.go:627-702) uses a worker pool (line 663)
- Creates worker goroutines with `Keys.NumWorkers` workers (line 663)
- Each worker processes serialization of metric data to binary snapshot files
- This is parallel I/O-heavy work with **NO TIMEOUT**
- On shutdown, this must complete fully before returning

---

### 4. **WAL STAGING WORKER POOL** (`pkg/metricstore/walCheckpoint.go:154-320`)

**Recent Change (commit 0325d9e8 - "fix: Increase throughput for WAL writers"):**

The change modified the WAL staging implementation to:
- **Before:** Flushed immediately after every batch of ~256 messages
- **After:** Uses a 5-second timer (`walFlushInterval`) for periodic flushing (line 123)
- Added `drainBatch()` function that processes up to 4096 messages per batch (line 278-296)

**Architecture:**
- Creates `Keys.NumWorkers` sharded worker goroutines (lines 162-319)
- Each shard owns a subset of hosts (determined by FNV-1a hash)
- Each worker maintains `hostFiles` map with open file handles
- Workers process messages from buffered channels (capacity: `65536/walNumShards`)

**Shutdown Sequence:**
1. Closes all WAL shard channels (main.go line 281-283)
2. Triggers `drain()` function on each worker (line 301)
3. Drain tries to process remaining messages in "drainBatch" pattern
4. **Problem:** If channels still have pending messages, `drain()` will block processing them all

---

### 5. **ARCHIVER SHUTDOWN** (`internal/archiver/archiveWorker.go:196-250`)

**Has proper timeout handling:**
```go
func Shutdown(timeout time.Duration) error {
    close(archiveChannel)           // line 226
    done := make(chan struct{})
    go func() {
        archivePending.Wait()       // Wait for pending jobs
        close(done)
    }()

    select {
    case <-done:                    // All jobs completed
        <-workerDone                // Wait for worker exit
        return nil
    case <-time.After(timeout):     // Timeout exceeded
        shutdownCancel()            // Cancel ongoing operations
        <-workerDone                // Wait for worker to exit
        return fmt.Errorf("...")
    }
}
```

**Called with 10s timeout** (server.go:441)

---

### 6. **TASKMANAGER SHUTDOWN** (`internal/taskmanager/taskManager.go:164-169`)

**Simple implementation:**
```go
func Shutdown() {
    if s != nil {
        s.Shutdown()  // gocron scheduler shutdown (blocking)
    }
}
```

**No timeout specified** - delegates to gocron scheduler

---

### 7. **NATS API WORKER POOL** (`internal/api/nats.go:83-157`)

**Architecture:**
- Creates `JobConcurrency` job worker goroutines (line 105-107, default 8)
- Creates `NodeConcurrency` node worker goroutines (line 108-110, default 2)
- Uses buffered channels for message routing
- **No explicit shutdown in the code** - relies on channel closure during context cancellation

**Goroutines:**
- `jobWorker()` processes job events (lines 116-120)
- `nodeWorker()` processes node state (lines 123-127)
- Created during `NewNatsAPI()` (lines 105-110)
- Workers run indefinitely until channel closes

---

### 8. **IDENTIFIED TIMEOUT VALUES**

| Component | Timeout | Location | Issue |
|-----------|---------|----------|-------|
| HTTP Server Shutdown | 30s | server.go:420 | Uses dedicated timeout, OK |
| Archiver Shutdown | 10s | server.go:441 | Explicit timeout, OK |
| Metricstore Shutdown | **NONE** | server.go:437 | **NO TIMEOUT - CAN BLOCK** |
| WAL Final Checkpoint | **NONE** | metricstore.go:298 | **NO TIMEOUT - BLOCKING I/O** |
| Taskmanager Shutdown | **NONE** | main.go:417 | Delegates to gocron, may block |

---

### 9. **ROOT CAUSE ANALYSIS**

**Why shutdown takes ~40s instead of ~10s:**

1. **metricstore.Shutdown()** is called without timeout (server.go:437)
2. This function performs `ToCheckpointWAL()` which:
   - Creates worker pool with `Keys.NumWorkers` workers
   - Each worker serializes potentially GB-scale metric data to binary files
   - Uses parallel I/O with `sync.WaitGroup` blocking until all workers complete
   - Can take 20-30+ seconds on large metric datasets
3. There's **no cancellation mechanism** if checkpoint takes too long
4. During shutdown, the HTTP server gets 30s timeout, but everything after metricstore shutdown gets no timeout

**Why the recent WAL commit might have worsened this:**
- The commit increased throughput by batching messages and using periodic flushing
- This accumulates more in-flight messages in channels
- On shutdown, the `drain()` function must process all accumulated messages
- This increases the backlog that must be drained during shutdown

---

### 10. **SHUTDOWN SEQUENCE TIMELINE**

```
Signal received (t=0)
↓
srv.Shutdown(ctx) starts (30s timeout on HTTP graceful shutdown)
    ├─ NATS client closes
    └─ HTTP server waits for ongoing requests
↓
util.FsWatcherShutdown() (no timeout)
↓
metricstore.Shutdown() starts (NO TIMEOUT) ⚠
    ├─ Cancels context (stops background workers)
    ├─ Closes WAL shard channels
    ├─ Calls ToCheckpointWAL() (BLOCKING, worker pool, I/O intensive) ⚠
    │   └─ Creates NumWorkers goroutines
    │   └─ Each serializes metric data to disk
    │   └─ Waits for all with sync.WaitGroup
    └─ Calls RotateWALFilesAfterShutdown()
↓
taskmanager.Shutdown() (no timeout)
    └─ gocron.Shutdown() (blocking)
↓
archiver.Shutdown(10s) (10s timeout)
↓
wg.Wait() for all goroutines (blocking)
↓
Done (t=~40s)
```

---

### **ABSOLUTE FILE PATHS & LINE NUMBERS**

**Shutdown Entry Points:**
- `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go:339-463` - runServer()
- `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go:402-418` - Signal handling
- `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/server.go:418-444` - Server.Shutdown()

**Metricstore Shutdown (CRITICAL SECTION):**
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go:273-310` - Shutdown()
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go:627-702` - ToCheckpointWAL()
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go:154-320` - WALStaging() worker pool
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go:322-340` - RotateWALFiles()

**Archiver Shutdown:**
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go:196-250` - Shutdown()
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go:71-92` - Start()
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/archiver/archiveWorker.go:110-171` - archivingWorker()

**NATS API Worker Pool:**
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go:83-113` - NewNatsAPI()
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go:105-110` - Worker goroutine creation
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/nats.go:116-127` - Worker functions

**Taskmanager Shutdown:**
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/taskManager.go:103-145` - Start()
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/taskManager.go:164-169` - Shutdown()

---

### **RECOMMENDATIONS**

1. **Add timeout to metricstore.Shutdown()** - Currently has no mechanism to interrupt if checkpoint takes too long
2. **Drain WAL channels with timeout during shutdown** - The drain() function can block indefinitely on channels
3. **Make final checkpoint optional or configurable** - For fast shutdowns, allow skipping the final checkpoint
4. **Monitor TOCheckpointWAL() performance** - This worker pool I/O operation is the bottleneck
5. **Add NATS worker pool shutdown** - Currently no explicit shutdown for NATS API workers
6. **Add taskmanager shutdown timeout** - Delegate to gocron may block indefinitely

agentId: a430c972b949ad362 (use SendMessage with to: 'a430c972b949ad362' to continue this agent)
<usage>total_tokens: 80252
tool_uses: 32
duration_ms: 66730</usage>

Prompt:
I need to understand the shutdown flow and worker pool implementation in this Go backend. The shutdown time has increased from ~10s to ~40s, possibly due to recently added worker pools.

Please find and trace:

1. The main shutdown flow in `cmd/cc-backend/` — how the server shuts down, what timeout is used, what cleanup happens
2. Any worker pool implementations — look for goroutine pools, worker patterns, especially recently added ones
3. NATS shutdown/cleanup code in `internal/api/nats.go`
4. The metricstore shutdown in `pkg/metricstore/`
5. Any hardcoded timeout values related to shutdown (look for context.WithTimeout, time.After, signal handling, etc.)

Be very thorough — search for "Shutdown", "shutdown", "graceful", "worker", "pool", "drain", "Close", "signal", "SIGTERM", "SIGINT" across the codebase. Report exact file paths, line numbers, and the relevant code snippets.


Let me read the critical shutdown code sections to verify the findings.


   415→	return nil
   416→}
   417→
   418→func (s *Server) Shutdown(ctx context.Context) {
   419→	// Create a shutdown context with timeout
   420→	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
   421→	defer cancel()
   422→
   423→	nc := nats.GetClient()
   424→	if nc != nil {
   425→		nc.Close()
   426→	}
   427→
   428→	// First shut down the server gracefully (waiting for all ongoing requests)
   429→	if err := s.server.Shutdown(shutdownCtx); err != nil {
   430→		cclog.Errorf("Server shutdown error: %v", err)
   431→	}
   432→
   433→	// Archive all the metric store data
   434→	ms := metricstore.GetMemoryStore()
   435→
   436→	if ms != nil {
   437→		metricstore.Shutdown()
   438→	}
   439→
   440→	// Shutdown archiver with 10 second timeout for fast shutdown
   441→	if err := archiver.Shutdown(10 * time.Second); err != nil {
   442→		cclog.Warnf("Archiver shutdown: %v", err)
   443→	}
   444→}
   445→

	return nil
}

func (s *Server) Shutdown(ctx context.Context) {
	// Create a shutdown context with timeout
	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	// First shut down the server gracefully (waiting for all ongoing requests)
	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Archive all the metric store data
	ms := metricstore.GetMemoryStore()

	if ms != nil {
		metricstore.Shutdown()
	}

	// Shutdown archiver with 10 second timeout for fast shutdown
	if err := archiver.Shutdown(10 * time.Second); err != nil {
		cclog.Warnf("Archiver shutdown: %v", err)
	}
}


   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→
   290→	ms := GetMemoryStore()
   291→
   292→	lastCheckpointMu.Lock()
   293→	from := lastCheckpoint
   294→	lastCheckpointMu.Unlock()
   295→
   296→	if Keys.Checkpoints.FileFormat == "wal" {
   297→		var hostDirs []string
   298→		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   299→		if err == nil {
   300→			RotateWALFilesAfterShutdown(hostDirs)
   301→		}
   302→	} else {
   303→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   304→	}
   305→
   306→	if err != nil {
   307→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
   308→	}
   309→	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
   310→}
   311→
   312→// Retention starts a background goroutine that periodically frees old metric data.
   313→//
   314→// This worker runs at half the retention interval and calls Free() to remove buffers

//  4. Log any errors encountered during shutdown
//
// Note: This function blocks until the final checkpoint is written.
func Shutdown() {
	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}

	if Keys.Checkpoints.FileFormat == "wal" {
		for _, ch := range walShardChs {
			close(ch)
		}
	}

	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		var hostDirs []string
		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
		if err == nil {
			RotateWALFilesAfterShutdown(hostDirs)
		}
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
}

// Retention starts a background goroutine that periodically frees old metric data.
//
// This worker runs at half the retention interval and calls Free() to remove buffers


   150→// WALStaging starts N sharded background goroutines that receive WALMessage items
   151→// and append binary WAL records to per-host current.wal files.
   152→// Each shard owns a subset of hosts (determined by hash of cluster+node),
   153→// parallelizing serialization and I/O while keeping per-host writes serial.
   154→func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
   155→	if Keys.Checkpoints.FileFormat == "json" {
   156→		return
   157→	}
   158→
   159→	walNumShards = max(Keys.NumWorkers, 1)
   160→	chBufSize := max(65536/walNumShards, 1024)
   161→
   162→	walShardChs = make([]chan *WALMessage, walNumShards)
   163→	walShardRotateChs = make([]chan walRotateReq, walNumShards)
   164→
   165→	for i := range walNumShards {
   166→		walShardChs[i] = make(chan *WALMessage, chBufSize)
   167→		walShardRotateChs[i] = make(chan walRotateReq, 64)
   168→	}
   169→
   170→	for i := range walNumShards {
   171→		msgCh := walShardChs[i]
   172→		rotateCh := walShardRotateChs[i]
   173→
   174→		wg.Go(func() {
   175→			hostFiles := make(map[string]*walFileState)
   176→
   177→			defer func() {
   178→				for _, ws := range hostFiles {
   179→					if ws.f != nil {
   180→						ws.w.Flush()
   181→						ws.f.Close()
   182→					}
   183→				}
   184→			}()
   185→
   186→			getOrOpenWAL := func(hostDir string) *walFileState {
   187→				ws, ok := hostFiles[hostDir]
   188→				if ok {
   189→					return ws
   190→				}
   191→
   192→				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
   193→					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
   194→					return nil
   195→				}
   196→
   197→				walPath := path.Join(hostDir, "current.wal")
   198→				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
   199→				if err != nil {
   200→					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
   201→					return nil
   202→				}
   203→
   204→				w := bufio.NewWriter(f)
   205→
   206→				// Write file header magic if file is new (empty).
   207→				info, err := f.Stat()
   208→				if err == nil && info.Size() == 0 {
   209→					var hdr [4]byte
   210→					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
   211→					if _, err := w.Write(hdr[:]); err != nil {
   212→						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
   213→						f.Close()
   214→						return nil
   215→					}
   216→				}
   217→
   218→				ws = &walFileState{f: f, w: w}
   219→				hostFiles[hostDir] = ws
   220→				return ws
   221→			}
   222→
   223→			processMsg := func(msg *WALMessage) {
   224→				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
   225→				ws := getOrOpenWAL(hostDir)
   226→				if ws == nil {
   227→					return
   228→				}
   229→				if err := writeWALRecordDirect(ws.w, msg); err != nil {
   230→					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
   231→				}
   232→				ws.dirty = true
   233→			}
   234→
   235→			processRotate := func(req walRotateReq) {
   236→				ws, ok := hostFiles[req.hostDir]
   237→				if ok && ws.f != nil {
   238→					ws.w.Flush()
   239→					ws.f.Close()
   240→					walPath := path.Join(req.hostDir, "current.wal")
   241→					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
   242→						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
   243→					}
   244→					delete(hostFiles, req.hostDir)
   245→				}
   246→				close(req.done)
   247→			}
   248→
   249→			flushDirty := func() {
   250→				for _, ws := range hostFiles {
   251→					if ws.dirty {
   252→						ws.w.Flush()
   253→						ws.dirty = false
   254→					}
   255→				}
   256→			}
   257→
   258→			drain := func() {
   259→				for {
   260→					select {
   261→					case msg, ok := <-msgCh:
   262→						if !ok {
   263→							return
   264→						}
   265→						processMsg(msg)
   266→					case req := <-rotateCh:
   267→						processRotate(req)
   268→					default:
   269→						flushDirty()
   270→						return
   271→					}
   272→				}
   273→			}
   274→
   275→			ticker := time.NewTicker(walFlushInterval)
   276→			defer ticker.Stop()
   277→
   278→			// drainBatch processes up to 4096 pending messages without blocking.
   279→			// Returns false if the channel was closed.
   280→			drainBatch := func() bool {
   281→				for range 4096 {
   282→					select {
   283→					case msg, ok := <-msgCh:
   284→						if !ok {
   285→							flushDirty()
   286→							return false
   287→						}
   288→						processMsg(msg)
   289→					case req := <-rotateCh:
   290→						processRotate(req)
   291→					default:
   292→						return true
   293→					}
   294→				}
   295→				return true
   296→			}
   297→
   298→			for {
   299→				select {
   300→				case <-ctx.Done():
   301→					drain()
   302→					return
   303→				case msg, ok := <-msgCh:
   304→					if !ok {
   305→						return
   306→					}
   307→					processMsg(msg)
   308→					if !drainBatch() {
   309→						return
   310→					}
   311→					// No flush here — timer handles periodic flushing.
   312→				case <-ticker.C:
   313→					flushDirty()
   314→				case req := <-rotateCh:
   315→					processRotate(req)
   316→				}
   317→			}
   318→		})
   319→	}
   320→}
   321→
   322→// RotateWALFiles sends rotation requests for the given host directories
   323→// and blocks until all rotations complete. Each request is routed to the
   324→// shard that owns the host directory.
   325→func RotateWALFiles(hostDirs []string) {
   326→	if walShardRotateChs == nil {
   327→		return
   328→	}
   329→	dones := make([]chan struct{}, len(hostDirs))

// WALStaging starts N sharded background goroutines that receive WALMessage items
// and append binary WAL records to per-host current.wal files.
// Each shard owns a subset of hosts (determined by hash of cluster+node),
// parallelizing serialization and I/O while keeping per-host writes serial.
func WALStaging(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Checkpoints.FileFormat == "json" {
		return
	}

	walNumShards = max(Keys.NumWorkers, 1)
	chBufSize := max(65536/walNumShards, 1024)

	walShardChs = make([]chan *WALMessage, walNumShards)
	walShardRotateChs = make([]chan walRotateReq, walNumShards)

	for i := range walNumShards {
		walShardChs[i] = make(chan *WALMessage, chBufSize)
		walShardRotateChs[i] = make(chan walRotateReq, 64)
	}

	for i := range walNumShards {
		msgCh := walShardChs[i]
		rotateCh := walShardRotateChs[i]

		wg.Go(func() {
			hostFiles := make(map[string]*walFileState)

			defer func() {
				for _, ws := range hostFiles {
					if ws.f != nil {
						ws.w.Flush()
						ws.f.Close()
					}
				}
			}()

			getOrOpenWAL := func(hostDir string) *walFileState {
				ws, ok := hostFiles[hostDir]
				if ok {
					return ws
				}

				if err := os.MkdirAll(hostDir, CheckpointDirPerms); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: mkdir %s: %v", hostDir, err)
					return nil
				}

				walPath := path.Join(hostDir, "current.wal")
				f, err := os.OpenFile(walPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, CheckpointFilePerms)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: open %s: %v", walPath, err)
					return nil
				}

				w := bufio.NewWriter(f)

				// Write file header magic if file is new (empty).
				info, err := f.Stat()
				if err == nil && info.Size() == 0 {
					var hdr [4]byte
					binary.LittleEndian.PutUint32(hdr[:], walFileMagic)
					if _, err := w.Write(hdr[:]); err != nil {
						cclog.Errorf("[METRICSTORE]> WAL: write header %s: %v", walPath, err)
						f.Close()
						return nil
					}
				}

				ws = &walFileState{f: f, w: w}
				hostFiles[hostDir] = ws
				return ws
			}

			processMsg := func(msg *WALMessage) {
				hostDir := path.Join(Keys.Checkpoints.RootDir, msg.Cluster, msg.Node)
				ws := getOrOpenWAL(hostDir)
				if ws == nil {
					return
				}
				if err := writeWALRecordDirect(ws.w, msg); err != nil {
					cclog.Errorf("[METRICSTORE]> WAL: write record: %v", err)
				}
				ws.dirty = true
			}

			processRotate := func(req walRotateReq) {
				ws, ok := hostFiles[req.hostDir]
				if ok && ws.f != nil {
					ws.w.Flush()
					ws.f.Close()
					walPath := path.Join(req.hostDir, "current.wal")
					if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
						cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
					}
					delete(hostFiles, req.hostDir)
				}
				close(req.done)
			}

			flushDirty := func() {
				for _, ws := range hostFiles {
					if ws.dirty {
						ws.w.Flush()
						ws.dirty = false
					}
				}
			}

			drain := func() {
				for {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							return
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						flushDirty()
						return
					}
				}
			}

			ticker := time.NewTicker(walFlushInterval)
			defer ticker.Stop()

			// drainBatch processes up to 4096 pending messages without blocking.
			// Returns false if the channel was closed.
			drainBatch := func() bool {
				for range 4096 {
					select {
					case msg, ok := <-msgCh:
						if !ok {
							flushDirty()
							return false
						}
						processMsg(msg)
					case req := <-rotateCh:
						processRotate(req)
					default:
						return true
					}
				}
				return true
			}

			for {
				select {
				case <-ctx.Done():
					drain()
					return
				case msg, ok := <-msgCh:
					if !ok {
						return
					}
					processMsg(msg)
					if !drainBatch() {
						return
					}
					// No flush here — timer handles periodic flushing.
				case <-ticker.C:
					flushDirty()
				case req := <-rotateCh:
					processRotate(req)
				}
			}
		})
	}
}

// RotateWALFiles sends rotation requests for the given host directories
// and blocks until all rotations complete. Each request is routed to the
// shard that owns the host directory.
func RotateWALFiles(hostDirs []string) {
	if walShardRotateChs == nil {
		return
	}
	dones := make([]chan struct{}, len(hostDirs))


   620→	}
   621→	return result.String()
   622→}
   623→
   624→// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
   625→// Returns the number of files written, the list of host directories that were
   626→// successfully checkpointed (for WAL rotation), and any errors.
   627→func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
   628→	// Collect all cluster/host pairs.
   629→	m.root.lock.RLock()
   630→	totalHosts := 0
   631→	for _, l1 := range m.root.children {
   632→		l1.lock.RLock()
   633→		totalHosts += len(l1.children)
   634→		l1.lock.RUnlock()
   635→	}
   636→	m.root.lock.RUnlock()
   637→
   638→	levels := make([]*Level, 0, totalHosts)
   639→	selectors := make([][]string, 0, totalHosts)
   640→
   641→	m.root.lock.RLock()
   642→	for sel1, l1 := range m.root.children {
   643→		l1.lock.RLock()
   644→		for sel2, l2 := range l1.children {
   645→			levels = append(levels, l2)
   646→			selectors = append(selectors, []string{sel1, sel2})
   647→		}
   648→		l1.lock.RUnlock()
   649→	}
   650→	m.root.lock.RUnlock()
   651→
   652→	type workItem struct {
   653→		level    *Level
   654→		hostDir  string
   655→		selector []string
   656→	}
   657→
   658→	n, errs := int32(0), int32(0)
   659→	var successDirs []string
   660→	var successMu sync.Mutex
   661→
   662→	var wg sync.WaitGroup
   663→	wg.Add(Keys.NumWorkers)
   664→	work := make(chan workItem, Keys.NumWorkers*2)
   665→
   666→	for range Keys.NumWorkers {
   667→		go func() {
   668→			defer wg.Done()
   669→			for wi := range work {
   670→				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
   671→				if err != nil {
   672→					if err == ErrNoNewArchiveData {
   673→						continue
   674→					}
   675→					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
   676→					atomic.AddInt32(&errs, 1)
   677→				} else {
   678→					atomic.AddInt32(&n, 1)
   679→					successMu.Lock()
   680→					successDirs = append(successDirs, wi.hostDir)
   681→					successMu.Unlock()
   682→				}
   683→			}
   684→		}()
   685→	}
   686→
   687→	for i := range levels {
   688→		hostDir := path.Join(dir, path.Join(selectors[i]...))
   689→		work <- workItem{
   690→			level:    levels[i],
   691→			hostDir:  hostDir,
   692→			selector: selectors[i],
   693→		}
   694→	}
   695→	close(work)
   696→	wg.Wait()
   697→
   698→	if errs > 0 {
   699→		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
   700→	}
   701→	return int(n), successDirs, nil
   702→}
   703→
   704→// toCheckpointBinary writes a binary snapshot file for a single host-level node.
   705→// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
   706→func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
   707→	cf, err := l.toCheckpointFile(from, to, m)
   708→	if err != nil {
   709→		return err

	}
	return result.String()
}

// ToCheckpointWAL writes binary snapshot files for all hosts in parallel.
// Returns the number of files written, the list of host directories that were
// successfully checkpointed (for WAL rotation), and any errors.
func (m *MemoryStore) ToCheckpointWAL(dir string, from, to int64) (int, []string, error) {
	// Collect all cluster/host pairs.
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		hostDir  string
		selector []string
	}

	n, errs := int32(0), int32(0)
	var successDirs []string
	var successMu sync.Mutex

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)

	for range Keys.NumWorkers {
		go func() {
			defer wg.Done()
			for wi := range work {
				err := wi.level.toCheckpointBinary(wi.hostDir, from, to, m)
				if err != nil {
					if err == ErrNoNewArchiveData {
						continue
					}
					cclog.Errorf("[METRICSTORE]> binary checkpoint error for %s: %v", wi.hostDir, err)
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
					successMu.Lock()
					successDirs = append(successDirs, wi.hostDir)
					successMu.Unlock()
				}
			}
		}()
	}

	for i := range levels {
		hostDir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			hostDir:  hostDir,
			selector: selectors[i],
		}
	}
	close(work)
	wg.Wait()

	if errs > 0 {
		return int(n), successDirs, fmt.Errorf("[METRICSTORE]> %d errors during binary checkpoint (%d successes)", errs, n)
	}
	return int(n), successDirs, nil
}

// toCheckpointBinary writes a binary snapshot file for a single host-level node.
// Uses atomic rename (write to .tmp then rename) to avoid partial reads on crash.
func (l *Level) toCheckpointBinary(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err


   395→	// Start HTTP server
   396→	wg.Go(func() {
   397→		if err := srv.Start(ctx); err != nil {
   398→			errChan <- err
   399→		}
   400→	})
   401→
   402→	// Handle shutdown signals
   403→	wg.Add(1)
   404→	sigs := make(chan os.Signal, 1)
   405→	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
   406→	go func() {
   407→		defer wg.Done()
   408→		select {
   409→		case <-sigs:
   410→			cclog.Info("Shutdown signal received")
   411→		case <-ctx.Done():
   412→		}
   413→
   414→		runtime.SystemdNotify(false, "Shutting down ...")
   415→		srv.Shutdown(ctx)
   416→		util.FsWatcherShutdown()
   417→		taskmanager.Shutdown()
   418→	}()
   419→
   420→	// Set GC percent if not configured
   421→	if os.Getenv(envGOGC) == "" {
   422→		// trigger GC when heap grows 15% above the previous live set
   423→		debug.SetGCPercent(15)
   424→	}
   425→	runtime.SystemdNotify(true, "running")
   426→
   427→	waitDone := make(chan struct{})
   428→	go func() {
   429→		wg.Wait()
   430→		close(waitDone)
   431→	}()
   432→
   433→	go func() {
   434→		<-waitDone
   435→		close(errChan)
   436→	}()
   437→
   438→	// Wait for either:
   439→	// 1. An error from server startup
   440→	// 2. Completion of all goroutines (normal shutdown or crash)
   441→	select {
   442→	case err := <-errChan:
   443→		// errChan will be closed when waitDone is closed, which happens
   444→		// when all goroutines complete (either from normal shutdown or error)

	// Start HTTP server
	wg.Go(func() {
		if err := srv.Start(ctx); err != nil {
			errChan <- err
		}
	})

	// Handle shutdown signals
	wg.Add(1)
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		defer wg.Done()
		select {
		case <-sigs:
			cclog.Info("Shutdown signal received")
		case <-ctx.Done():
		}

		runtime.SystemdNotify(false, "Shutting down ...")
		srv.Shutdown(ctx)
		util.FsWatcherShutdown()
		taskmanager.Shutdown()
	}()

	// Set GC percent if not configured
	if os.Getenv(envGOGC) == "" {
		// trigger GC when heap grows 15% above the previous live set
		debug.SetGCPercent(15)
	}
	runtime.SystemdNotify(true, "running")

	waitDone := make(chan struct{})
	go func() {
		wg.Wait()
		close(waitDone)
	}()

	go func() {
		<-waitDone
		close(errChan)
	}()

	// Wait for either:
	// 1. An error from server startup
	// 2. Completion of all goroutines (normal shutdown or crash)
	select {
	case err := <-errChan:
		// errChan will be closed when waitDone is closed, which happens
		// when all goroutines complete (either from normal shutdown or error)


pkg/metricstore/metricstore.go:300:			RotateWALFilesAfterShutdown(hostDirs)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-301-		}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-302-	} else {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-303-		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-304-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-305-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-306-	if err != nil {
pkg/metricstore/metricstore.go-307-		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-308-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-309-	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-310-}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-311-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-312-// Retention starts a background goroutine that periodically frees old metric data.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-313-//
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-314-// This worker runs at half the retention interval and calls Free() to remove buffers
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-315-// older than the configured retention time. It respects the NodeProvider to preserve
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-316-// data for nodes with active jobs.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-317-//
pkg/metricstore/metricstore.go-318-/ Parameters:
pkg/metricstore/metricstore.go-319-/   - wg: WaitGroup to signal completion when context is cancelled
pkg/metricstore/metricstore.go-320-/   - ctx: Context for cancellation signal
--
pkg/metricstore/walCheckpoint.go:353:func RotateWALFilesAfterShutdown(hostDirs []string) {
pkg/metricstore/walCheckpoint.go-354-	for _, dir := range hostDirs {
pkg/metricstore/walCheckpoint.go-355-		walPath := path.Join(dir, "current.wal")
pkg/metricstore/walCheckpoint.go-356-		if err := os.Remove(walPath); err != nil && !os.IsNotExist(err) {
pkg/metricstore/walCheckpoint.go-357-			cclog.Errorf("[METRICSTORE]> WAL: remove %s: %v", walPath, err)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-358-		}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-359-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-360-}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-361-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-362-// writeWALRecordDirect encodes a WAL record directly into the bufio.Writer,
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-363-// avoiding heap allocations by using a stack-allocated scratch buffer for
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-364-// the fixed-size header/trailer and computing CRC inline.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-365-func writeWALRecordDirect(w *bufio.Writer, msg *WALMessage) error {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-366-	// Compute payload size.
pkg/metricstore/walCheckpoint.go-367-	payloadSize := 8 + 2 + len(msg.MetricName) + 1 + 4
pkg/metricstore/walCheckpoint.go-368-	for _, s := range msg.Selector {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-369-		payloadSize += 1 + len(s)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-370-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-371-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-372-	// Write magic + payload length (8 bytes header).
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-373-	var hdr [8]byte


Now I have a clear picture. Let me also check how the WAL shard channels interact with the shutdown — specifically, the order matters: channels are closed (line 281-283) before `ToCheckpointWAL` is called (line 298), and the `wg.Wait()` in main.go waits for the WAL staging goroutines to finish.


   220→		}
   221→
   222→		msInstance = &MemoryStore{
   223→			root: Level{
   224→				metrics:  make([]*buffer, len(metrics)),
   225→				children: make(map[string]*Level),
   226→			},
   227→			Metrics: metrics,
   228→		}
   229→	})
   230→}
   231→
   232→// GetMemoryStore returns the singleton MemoryStore instance.
   233→//
   234→// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
   235→//
   236→// This function is safe for concurrent use after initialization.
   237→func GetMemoryStore() *MemoryStore {
   238→	if msInstance == nil {
   239→		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
   240→	}
   241→
   242→	return msInstance
   243→}
   244→
   245→func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
   246→	if metric, ok := ms.Metrics[metricName]; ok {
   247→		return metric.Frequency, nil
   248→	}
   249→	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
   250→}
   251→
   252→// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
   253→// This must be called during initialization to provide job state information
   254→// for selective buffer retention during Free operations.
   255→// If not set, the Free function will fall back to freeing all buffers.
   256→func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
   257→	ms.nodeProvider = provider
   258→}
   259→
   260→// Shutdown performs a graceful shutdown of the metric store.
   261→//
   262→// This function cancels all background goroutines started by Init() and writes
   263→// a final checkpoint to disk before returning. It should be called when the
   264→// application receives a termination signal.
   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()

		}

		msInstance = &MemoryStore{
			root: Level{
				metrics:  make([]*buffer, len(metrics)),
				children: make(map[string]*Level),
			},
			Metrics: metrics,
		}
	})
}

// GetMemoryStore returns the singleton MemoryStore instance.
//
// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
//
// This function is safe for concurrent use after initialization.
func GetMemoryStore() *MemoryStore {
	if msInstance == nil {
		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
	}

	return msInstance
}

func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
	if metric, ok := ms.Metrics[metricName]; ok {
		return metric.Frequency, nil
	}
	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
}

// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
// This must be called during initialization to provide job state information
// for selective buffer retention during Free operations.
// If not set, the Free function will fall back to freeing all buffers.
func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
	ms.nodeProvider = provider
}

// Shutdown performs a graceful shutdown of the metric store.
//
// This function cancels all background goroutines started by Init() and writes
// a final checkpoint to disk before returning. It should be called when the
// application receives a termination signal.
//
// The function will:
//  1. Cancel the context to stop all background workers
//  2. Close the WAL messages channel if using WAL format
//  3. Write a final checkpoint to preserve in-memory data
//  4. Log any errors encountered during shutdown
//
// Note: This function blocks until the final checkpoint is written.
func Shutdown() {
	shutdownFuncMu.Lock()


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-48-var (
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-49-	singleton  sync.Once
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-50-	msInstance *MemoryStore
pkg/metricstore/metricstore.go:51:	// shutdownFunc stores the context cancellation function created in Init
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-52-	// and is called during Shutdown to cancel all background goroutines
pkg/metricstore/metricstore.go:53:	shutdownFunc   context.CancelFunc
pkg/metricstore/metricstore.go:54:	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-55-	// Create a global instance
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-56-	state = &GlobalState{}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-57-)
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-179-	// The caller is responsible for handling shutdown signals and calling
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-180-	// the shutdown() function when appropriate.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-181-	// Store the shutdown function for later use by Shutdown()
pkg/metricstore/metricstore.go:182:	shutdownFuncMu.Lock()
pkg/metricstore/metricstore.go:183:	shutdownFunc = shutdown
pkg/metricstore/metricstore.go:184:	shutdownFuncMu.Unlock()
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-185-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-186-	if Keys.Subscriptions != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-187-		wg.Go(func() {
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-271-//
pkg/metricstore/metricstore.go-272-/ Note: This function blocks until the final checkpoint is written.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-273-func Shutdown() {
pkg/metricstore/metricstore.go:274:	shutdownFuncMu.Lock()
pkg/metricstore/metricstore.go:275:	defer shutdownFuncMu.Unlock()
pkg/metricstore/metricstore.go:276:	if shutdownFunc != nil {
pkg/metricstore/metricstore.go:277:		shutdownFunc()
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-278-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-279-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-280-	if Keys.Checkpoints.FileFormat == "wal" {


Now I have the full picture. Here's the critical issue:

**In `Shutdown()` (metricstore.go:273-310):**
1. Line 277: `shutdownFunc()` — cancels the context
2. Lines 281-283: closes all WAL shard channels
3. Line 298: calls `ToCheckpointWAL()` — writes binary snapshots for ALL hosts

But step 1 triggers `ctx.Done()` in the WAL staging goroutines, which calls `drain()` (line 301 in walCheckpoint.go). The `drain()` processes remaining messages and then the goroutine returns. Meanwhile, step 3 then writes binary snapshots for every host — this is the slow part, and it happens **after** the WAL workers already drained.

The real bottleneck is `ToCheckpointWAL()` writing binary snapshots for every host with no timeout. But the question is whether this was always happening. Let me check the `wg` interaction — the WAL staging goroutines are on the same `wg` as the shutdown handler.