	finalPath := path.Join(dir, fmt.Sprintf("%d.json", from))
	tmpPath := finalPath + ".tmp"

	f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil && os.IsNotExist(err) {
		err = os.MkdirAll(dir, CheckpointDirPerms)
		if err == nil {
			f, err = os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
		}
	}
	if err != nil {
		return err
	}

	bw := bufio.NewWriter(f)
	if err = json.NewEncoder(bw).Encode(cf); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return err
	}

	if err = bw.Flush(); err != nil {
		f.Close()
		os.Remove(tmpPath)
		return err
	}
	f.Close()

	return os.Rename(tmpPath, finalPath)

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements checkpoint persistence for the in-memory metric store.
//
// Checkpoints enable graceful restarts by periodically saving in-memory metric
// data to disk in JSON or binary format. The checkpoint system:
//
// Key Features:
//   - Periodic background checkpointing via the Checkpointing() worker
//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
//   - Parallel checkpoint creation and loading using worker pools
//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
//   - Only saves unarchived data (archived data is already persisted elsewhere)
//   - GC optimization during loading to prevent excessive heap growth
//
// Checkpoint Workflow:
//  1. Init() loads checkpoints within retention window at startup
//  2. Checkpointing() worker periodically saves new data
//  3. Shutdown() writes final checkpoint before exit
//
// File Organization:
//
//	checkpoints/
//	  cluster1/
//	    host001/
//	      1234567890.json  (JSON format: full subtree snapshot)
//	      1234567890.bin   (binary format: full subtree snapshot)
//	      current.wal      (WAL format: append-only per-entry log)
//	    host002/
//	      ...
package metricstore

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

const (
	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
)

// CheckpointMetrics represents metric data in a checkpoint file.
// Whenever the structure changes, update MarshalJSON as well!
type CheckpointMetrics struct {
	Data      []schema.Float `json:"data"`
	Frequency int64          `json:"frequency"`
	Start     int64          `json:"start"`
}

// CheckpointFile represents the hierarchical structure of a checkpoint file.
// It mirrors the Level tree structure from the MemoryStore.
type CheckpointFile struct {
	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
	Children map[string]*CheckpointFile    `json:"children"`
	From     int64                         `json:"from"`
	To       int64                         `json:"to"`
}

// lastCheckpoint tracks the timestamp of the last checkpoint creation.
var (
	lastCheckpoint   time.Time
	lastCheckpointMu sync.Mutex
)

// Checkpointing starts a background worker that periodically saves metric data to disk.
//
// restoreFrom is the earliest timestamp of data loaded from checkpoint files at startup.
// The first periodic checkpoint after restart will cover [restoreFrom, now], ensuring that
// loaded data is re-persisted before old checkpoint files are cleaned up.
//
// Format behaviour:
//   - "json": Periodic checkpointing every checkpointInterval
//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
func Checkpointing(wg *sync.WaitGroup, ctx context.Context, restoreFrom time.Time) {
	lastCheckpointMu.Lock()
	lastCheckpoint = restoreFrom
	lastCheckpointMu.Unlock()

	ms := GetMemoryStore()

	wg.Go(func() {

		d := 12 * time.Hour // default checkpoint interval
		if Keys.CheckpointInterval != "" {
			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
			if err != nil {
				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
			} else {
				d = parsed
			}
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				lastCheckpointMu.Lock()
				from := lastCheckpoint
				lastCheckpointMu.Unlock()

				now := time.Now()
				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))

				if Keys.Checkpoints.FileFormat == "wal" {
					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
					} else {
						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
						// Rotate WAL files for successfully checkpointed hosts.
						RotateWALFiles(hostDirs)
					}
				} else {
					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
					}
				}
			}
		}
	})
}

// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
//
// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
// This method manually constructs JSON to avoid allocations and interface conversions.
func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
	buf := make([]byte, 0, 128+len(cm.Data)*8)
	buf = append(buf, `{"frequency":`...)
	buf = strconv.AppendInt(buf, cm.Frequency, 10)
	buf = append(buf, `,"start":`...)
	buf = strconv.AppendInt(buf, cm.Start, 10)
	buf = append(buf, `,"data":[`...)
	for i, x := range cm.Data {
		if i != 0 {
			buf = append(buf, ',')
		}
		if x.IsNaN() {
			buf = append(buf, `null`...)
		} else {
			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
		}
	}
	buf = append(buf, `]}`...)
	return buf, nil
}

// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format).
//
// Metrics at root and cluster levels are skipped. One file per host is created.
// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
// at a time, allowing concurrent writes/reads to other hosts.
//
// Returns the number of checkpoint files created and any errors encountered.
func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
	// Pre-calculate capacity by counting cluster/host pairs
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		dir      string
		selector []string
	}

	n, errs := int32(0), int32(0)

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()

			for workItem := range work {
				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
					if err == ErrNoNewArchiveData {
						continue
					}

					cclog.Errorf("[METRICSTORE]> error while checkpointing %#v: %s", workItem.selector, err.Error())
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
				}
			}
		}()
	}

	for i := 0; i < len(levels); i++ {
		dir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			dir:      dir,
			selector: selectors[i],
		}
	}

	close(work)
	wg.Wait()

	if errs > 0 {
		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
	}
	return int(n), nil
}

// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
// Skips metrics that are already archived. Returns nil if no unarchived data exists.
func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	retval := &CheckpointFile{
		From:     from,
		To:       to,
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	for metric, minfo := range m.Metrics {
		b := l.metrics[minfo.offset]
		if b == nil {
			continue
		}

		allArchived := true
		b.iterFromTo(from, to, func(b *buffer) error {
			if !b.archived {
				allArchived = false
				return fmt.Errorf("stop") // Early termination signal
			}
			return nil
		})

		if allArchived {
			continue
		}

		data := make([]schema.Float, (to-from)/b.frequency+1)
		data, start, end, err := b.read(from, to, data)
		if err != nil {
			return nil, err
		}

		for i := int((end - start) / b.frequency); i < len(data); i++ {
			data[i] = schema.NaN
		}

		retval.Metrics[metric] = &CheckpointMetrics{
			Frequency: b.frequency,
			Start:     start,
			Data:      data,
		}
	}

	for name, child := range l.children {
		val, err := child.toCheckpointFile(from, to, m)
		if err != nil {
			return nil, err
		}

		if val != nil {
			retval.Children[name] = val
		}
	}

	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
		return nil, nil
	}

	return retval, nil
}

// toCheckpoint writes a Level's data to a JSON checkpoint file.
// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.
func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}

	if cf == nil {
		return ErrNoNewArchiveData
	}

	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil && os.IsNotExist(err) {
		err = os.MkdirAll(dir, CheckpointDirPerms)
		if err == nil {
			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
		}
	}
	if err != nil {
		return err
	}
	defer f.Close()

	bw := bufio.NewWriter(f)
	if err = json.NewEncoder(bw).Encode(cf); err != nil {
		return err
	}

	return bw.Flush()
}

// enqueueCheckpointHosts traverses checkpoint directory and enqueues cluster/host pairs.
// Returns error if directory structure is invalid.
func enqueueCheckpointHosts(dir string, work chan<- [2]string) error {
	clustersDir, err := os.ReadDir(dir)
	if err != nil {
		return err
	}

	for _, clusterDir := range clustersDir {
		if !clusterDir.IsDir() {
			return errors.New("[METRICSTORE]> expected only directories at first level of checkpoints/ directory")
		}

		hostsDir, err := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
		if err != nil {
			return err
		}

		for _, hostDir := range hostsDir {
			if !hostDir.IsDir() {
				return errors.New("[METRICSTORE]> expected only directories at second level of checkpoints/ directory")
			}

			work <- [2]string{clusterDir.Name(), hostDir.Name()}
		}
	}

	return nil
}

// FromCheckpoint loads checkpoint files from disk into memory in parallel.
//
// Uses worker pool to load cluster/host combinations. Returns number of files
// loaded and any errors.
func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
	var wg sync.WaitGroup
	work := make(chan [2]string, Keys.NumWorkers*4)
	n, errs := int32(0), int32(0)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for host := range work {
				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
					atomic.AddInt32(&errs, 1)
				}
				atomic.AddInt32(&n, int32(nn))
			}
		}()
	}

	err := enqueueCheckpointHosts(dir, work)
	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}

	if errs > 0 {
		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
	}
	return int(n), nil
}

// FromCheckpointFiles is the main entry point for loading checkpoints at startup.
//
// Creates checkpoint directory if it doesn't exist. This function must be called
// before any writes or reads, and can only be called once.
func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, CheckpointDirPerms)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
		}
		cclog.Debugf("[METRICSTORE]> %#v Directory created successfully", dir)
	}

	return m.FromCheckpoint(dir, from)
}

func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)
	cf := &CheckpointFile{}
	if err := json.NewDecoder(br).Decode(cf); err != nil {
		return err
	}

	if cf.To != 0 && cf.To < from {
		return nil
	}

	if err := l.loadFile(cf, m); err != nil {
		return err
	}

	return nil
}

func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
	for name, metric := range cf.Metrics {
		n := len(metric.Data)
		b := &buffer{
			frequency: metric.Frequency,
			start:     metric.Start,
			data:      metric.Data[0:n:n],
			prev:      nil,
			next:      nil,
			archived:  false,
		}

		minfo, ok := m.Metrics[name]
		if !ok {
			continue
		}

		prev := l.metrics[minfo.offset]
		if prev == nil {
			l.metrics[minfo.offset] = b
		} else {
			if prev.start > b.start {
				return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start)
			}

			b.prev = prev
			prev.next = b
		}
		l.metrics[minfo.offset] = b
	}

	if len(cf.Children) > 0 && l.children == nil {
		l.children = make(map[string]*Level)
	}

	for sel, childCf := range cf.Children {
		child, ok := l.children[sel]
		if !ok {
			child = &Level{
				metrics:  make([]*buffer, len(m.Metrics)),
				children: nil,
			}
			l.children[sel] = child
		}

		if err := child.loadFile(childCf, m); err != nil {
			return err
		}
	}

	return nil
}

// fromCheckpoint loads all checkpoint files (JSON, binary snapshot, WAL) for a
// single host directory. Snapshot files are loaded first (sorted by timestamp),
// then current.wal is replayed on top.
func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {
	direntries, err := os.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return 0, nil
		}
		return 0, err
	}

	allFiles := make([]fs.DirEntry, 0)
	var walEntry fs.DirEntry
	filesLoaded := 0

	for _, e := range direntries {
		if e.IsDir() {
			// Legacy: skip subdirectories (only used by old Avro format).
			// These are ignored; their data is not loaded.
			cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir)
			continue
		}

		name := e.Name()
		if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") {
			allFiles = append(allFiles, e)
		} else if name == "current.wal" {
			walEntry = e
		}
		// Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes).
	}

	files, err := findFiles(allFiles, from, true)
	if err != nil {
		return filesLoaded, err
	}

	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
		".json": l.loadJSONFile,
		".bin":  l.loadBinaryFile,
	}

	for _, filename := range files {
		ext := filepath.Ext(filename)
		loader := loaders[ext]
		if loader == nil {
			cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename)
			continue
		}

		err := func() error {
			f, err := os.Open(path.Join(dir, filename))
			if err != nil {
				return err
			}
			defer f.Close()
			return loader(m, f, from)
		}()
		if err != nil {
			return filesLoaded, err
		}
		filesLoaded++
	}

	// Replay WAL after all snapshot files so it fills in data since the last snapshot.
	if walEntry != nil {
		err := func() error {
			f, err := os.Open(path.Join(dir, walEntry.Name()))
			if err != nil {
				return err
			}
			defer f.Close()
			return l.loadWALFile(m, f, from)
		}()
		if err != nil {
			// WAL errors are non-fatal: the snapshot already loaded the bulk of data.
			cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err)
		} else {
			filesLoaded++
		}
	}

	return filesLoaded, nil
}

// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename.
// Supports ".json" (format: "<ts>.json") and ".bin" (format: "<ts>.bin").
func parseTimestampFromFilename(name string) (int64, error) {
	switch {
	case strings.HasSuffix(name, ".json"):
		return strconv.ParseInt(name[:len(name)-5], 10, 64)
	case strings.HasSuffix(name, ".bin"):
		return strconv.ParseInt(name[:len(name)-4], 10, 64)
	default:
		return 0, fmt.Errorf("unknown checkpoint extension for file %q", name)
	}
}

// findFiles returns filenames from direntries whose timestamps satisfy the filter.
// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the
// last file before t if t falls between two files).
func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {
	nums := map[string]int64{}
	for _, e := range direntries {
		name := e.Name()
		if !strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".bin") {
			continue
		}

		ts, err := parseTimestampFromFilename(name)
		if err != nil {
			return nil, err
		}
		nums[name] = ts
	}

	sort.Slice(direntries, func(i, j int) bool {
		a, b := direntries[i], direntries[j]
		return nums[a.Name()] < nums[b.Name()]
	})

	if len(nums) == 0 {
		return nil, nil
	}

	filenames := make([]string, 0)

	for i, e := range direntries {
		ts1 := nums[e.Name()]

		if findMoreRecentFiles && t <= ts1 {
			filenames = append(filenames, e.Name())
		} else if !findMoreRecentFiles && ts1 <= t && ts1 != 0 {
			filenames = append(filenames, e.Name())
		}

		if i == len(direntries)-1 {
			continue
		}

		enext := direntries[i+1]
		ts2 := nums[enext.Name()]

		if findMoreRecentFiles {
			if ts1 < t && t < ts2 {
				filenames = append(filenames, e.Name())
			}
		}
	}

	return filenames, nil
}

-  filepath := path.Join(dir, fmt.Sprintf("%d.json", from))

-  f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)

+  finalPath := path.Join(dir, fmt.Sprintf("%d.json", from))

+  f, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)

-      f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)

+      f, err = os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)


## Fix 4: CleanUp uses `to` timestamp for file age

The `findFiles` function uses the filename timestamp (which is `from`) to determine age. But files cover `[from, to]`, and `to` could still be within the retention window. The cleanest fix is in the `cleanUpWorker` where we compute the cutoff time — we should subtract one checkpoint interval so that files whose `to` is still within retention aren't deleted.

Actually, let me re-read the cleanup logic more carefully. The `cleanUpWorker` computes `t := time.Now().Add(-d)` where `d` is the retention duration, then calls `CleanupCheckpoints(..., t.Unix(), ...)`. The `findFiles` with `findMoreRecentFiles=false` finds files where `ts1 <= t`. So a file named `{from}.bin` covering `[from, from+12h]` gets deleted when `from <= now - retention`, even though `from+12h` might still be within retention.

The fix: we should account for the checkpoint interval when computing the cleanup cutoff. Let me check how the cleanup interval relates to checkpoint interval.

Looking at `cleanUpWorker`, it uses `Keys.RetentionInMemory` as the interval for the ticker AND for computing the cutoff. That means cleanup runs every retention period and deletes files older than retention. The issue is that files cover a span.

The simplest fix: subtract one checkpoint interval from the cutoff in `deleteCheckpoints`/`archiveCheckpoints`. But we don't have easy access to the checkpoint interval there. A better approach: change `findFiles` to also accept a `maxSpan` parameter... but that's complex.

Actually, the simplest and most correct fix is: in the cleanup worker, subtract the checkpoint interval from the cutoff time. Let me check where the checkpoint interval is accessible.


pkg/metricstore/checkpoint.go:106:		if Keys.CheckpointInterval != "" {
pkg/metricstore/checkpoint.go:107:			parsed, err := time.ParseDuration(Keys.CheckpointInterval)
pkg/metricstore/checkpoint.go:109:				cclog.Errorf("[METRICSTORE]> invalid checkpoint-interval %q: %s, using default 12h", Keys.CheckpointInterval, err)
pkg/metricstore/config.go:126:	CheckpointInterval string         `json:"checkpoint-interval,omitempty"`


The checkpoint interval is accessible via `Keys.CheckpointInterval`. Let me update the cleanup worker to account for this span.


    46→func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
    47→	wg.Go(func() {
    48→		d, err := time.ParseDuration(interval)
    49→		if err != nil {
    50→			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
    51→		}
    52→		if d <= 0 {
    53→			return
    54→		}
    55→
    56→		ticker := time.NewTicker(d)
    57→		defer ticker.Stop()
    58→
    59→		for {
    60→			select {
    61→			case <-ctx.Done():
    62→				return
    63→			case <-ticker.C:
    64→				t := time.Now().Add(-d)
    65→				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))
    66→
    67→				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)
    68→
    69→				if err != nil {
    70→					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
    71→				} else {
    72→					if delete {
    73→						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
    74→					} else {
    75→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
    76→					}
    77→				}
    78→			}
    79→		}
    80→	})
    81→}

func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}


The issue: `t := time.Now().Add(-d)` uses the retention duration. A checkpoint file named `{from}.bin` covers data up to `from + checkpointInterval`. We should only delete files where `from + checkpointInterval <= now - retention`, i.e., `from <= now - retention - checkpointInterval`. So we need to subtract an additional checkpoint interval from the cutoff.


Edit

func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		// Account for checkpoint span: files named {from}.bin contain data up to
		// from+checkpointInterval. Subtract the checkpoint interval so we don't
		// delete files whose data still falls within the retention window.
		checkpointSpan := 12 * time.Hour
		if Keys.CheckpointInterval != "" {
			if parsed, err := time.ParseDuration(Keys.CheckpointInterval); err == nil {
				checkpointSpan = parsed
			}
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d).Add(-checkpointSpan)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))


The file /Users/jan/tmp/cc-backend/pkg/metricstore/archive.go has been updated successfully.

func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		// Account for checkpoint span: files named {from}.bin contain data up to
		// from+checkpointInterval. Subtract the checkpoint interval so we don't
		// delete files whose data still falls within the retention window.
		checkpointSpan := 12 * time.Hour
		if Keys.CheckpointInterval != "" {
			if parsed, err := time.ParseDuration(Keys.CheckpointInterval); err == nil {
				checkpointSpan = parsed
			}
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d).Add(-checkpointSpan)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0
	var clusterErrors []string

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			cclog.Errorf("[METRICSTORE]> %d errors reading checkpoints for cluster %s", errs, cluster)
			clusterErrors = append(clusterErrors, cluster)
			os.Remove(parquetFile)
			continue
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			cclog.Errorf("[METRICSTORE]> error writing parquet archive for cluster %s: %s", cluster, writeErr.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)

	if len(clusterErrors) > 0 {
		return totalFiles, fmt.Errorf("archiving failed for clusters: %s", strings.Join(clusterErrors, ", "))
	}

	return totalFiles, nil
}

+    // Account for checkpoint span: files named {from}.bin contain data up to

+    // from+checkpointInterval. Subtract the checkpoint interval so we don't

+    // delete files whose data still falls within the retention window.

+      if parsed, err := time.ParseDuration(Keys.CheckpointInterval); err == nil {

+        t := time.Now().Add(-d).Add(-checkpointSpan)


Now let me add the `time` import to `archive.go` — let me check if it's already there.


     8→import (
     9→	"context"
    10→	"errors"
    11→	"fmt"
    12→	"os"
    13→	"path/filepath"
    14→	"strings"
    15→	"sync"
    16→	"sync/atomic"
    17→	"time"
    18→
    19→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    20→)
    21→

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)


Good, `time` is already imported. Now let me check if there are any tests that call `Checkpointing` directly.


pkg/metricstore/checkpoint.go:12://   - Periodic background checkpointing via the Checkpointing() worker
pkg/metricstore/checkpoint.go:22://  2. Checkpointing() worker periodically saves new data
pkg/metricstore/checkpoint.go:96:func Checkpointing(wg *sync.WaitGroup, ctx context.Context, restoreFrom time.Time) {
pkg/metricstore/config.go:137:// Accessed by Init(), Checkpointing(), and other lifecycle functions.
pkg/metricstore/metricstore.go:173:	Checkpointing(wg, ctx, restoreFrom)


Only called from `Init`, no tests to update. Let me verify the build compiles.


Bash

Build metricstore package


(Bash completed with no output)


Clean build. Let me also verify the full project builds.


Build entire project


Let me run the tests.


Run metricstore tests


ok  	github.com/ClusterCockpit/cc-backend/pkg/metricstore	0.440s


All tests pass. Here's a summary of the four fixes:

### Fix 1 (Primary): Prevent data loss after restart
- **`checkpoint.go:96`**: `Checkpointing()` now accepts a `restoreFrom time.Time` parameter and initializes `lastCheckpoint = restoreFrom` instead of `time.Now()`. This ensures the first periodic checkpoint covers the entire loaded retention window, not just data since restart.
- **`checkpoint.go:473`**: Loaded buffers now set `archived: false` instead of `true`. This allows `toCheckpointFile` to include loaded data in new checkpoints, ensuring it gets re-persisted before old files are cleaned up.
- **`metricstore.go:173`**: Passes `restoreFrom` to `Checkpointing()`.

**Trade-off**: The first checkpoint after restart is larger (covers full retention window). This is a one-time cost for correctness.

### Fix 3: JSON checkpoint atomic writes
- **`checkpoint.go:340-358`**: JSON checkpoints now write to a `.json.tmp` file and `os.Rename` to the final path, matching the binary format's crash-safe pattern.

### Fix 4: CleanUp accounts for checkpoint span
- **`archive.go:64`**: The cleanup cutoff time now subtracts the checkpoint interval (default 12h) in addition to the retention duration. This prevents premature deletion of files whose `from` timestamp is outside retention but whose data (`to`) is still within retention.

### Bug 2 (not fixed)
The partial checkpoint failure / `lastCheckpoint` not updated issue was noted as medium severity. The current behavior (re-checkpointing from the old `from` time on retry) is actually conservative — it ensures no data is missed. The redundant writes are a performance cost but not a correctness issue.