Implement the following plan:

# Fix: Continue Archiving Other Clusters on Per-Cluster Errors

## Context

In `archiveCheckpoints()` (`pkg/metricstore/archive.go:174`), the outer loop iterates over clusters. Four error paths inside this loop use `return`, which aborts archiving for **all** remaining clusters when one cluster fails. Errors should be logged and the loop should continue to the next cluster.

## Error paths to fix (all inside the `for _, clusterEntry` loop)

1. **Line 193** — `os.ReadDir(hostEntries)` fails → `return totalFiles, err`
2. **Line 258** — `newParquetArchiveWriter()` fails → `return totalFiles, fmt.Errorf(...)`
3. **Line 294** — worker `errs > 0` → `return totalFiles, fmt.Errorf(...)`
4. **Line 305** — `writeErr != nil` → `return totalFiles, fmt.Errorf(...)`

## Fix — `pkg/metricstore/archive.go`

Replace all 4 early `return` statements with `cclog.Errorf(...)` + `continue` so the loop proceeds to the next cluster. Track whether any cluster had errors and return a summary error at the end (after all clusters have been attempted).

Specifically:
- Add `var clusterErrors []string` before the loop
- Each of the 4 error paths: log the error, append to `clusterErrors`, `continue`
- After the loop: if `len(clusterErrors) > 0`, return a joined error summary
- The drain-and-continue logic for the parquet writer failure (path 2) already exists and just needs `continue` instead of `return`
- For paths 3 and 4: the parquet file cleanup (`os.Remove`) already happens, just need `continue` instead of `return`
- For path 1: no cleanup needed, just skip this cluster

## Verification

- `go build ./pkg/metricstore/...`
- `go test ./pkg/metricstore/...`

If you need specific details from before exiting plan mode (like exact code snippets, error messages, or content you generated), read the full transcript at: /Users/jan/.claude/projects/-Users-jan-prg-ClusterCockpit-cc-backend/0696ef34-7ff1-4c1e-9103-ffbca94f3ebd.jsonl


   170→// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
   171→// Workers load checkpoint files from disk and send CheckpointFile trees on a
   172→// back-pressured channel. The main thread streams each tree directly to Parquet
   173→// rows without materializing all rows in memory.
   174→func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
   175→	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
   176→	startTime := time.Now()
   177→
   178→	clusterEntries, err := os.ReadDir(checkpointsDir)
   179→	if err != nil {
   180→		return 0, err
   181→	}
   182→
   183→	totalFiles := 0
   184→
   185→	for _, clusterEntry := range clusterEntries {
   186→		if !clusterEntry.IsDir() {
   187→			continue
   188→		}
   189→
   190→		cluster := clusterEntry.Name()
   191→		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
   192→		if err != nil {
   193→			return totalFiles, err
   194→		}
   195→
   196→		// Workers load checkpoint files from disk; main thread writes to parquet.
   197→		type hostResult struct {
   198→			checkpoints []*CheckpointFile
   199→			hostname    string
   200→			files       []string // checkpoint filenames to delete after successful write
   201→			dir         string   // checkpoint directory for this host
   202→		}
   203→
   204→		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
   205→		results := make(chan hostResult, 2)
   206→		work := make(chan struct {
   207→			dir, host string
   208→		}, Keys.NumWorkers)
   209→
   210→		var wg sync.WaitGroup
   211→		errs := int32(0)
   212→
   213→		wg.Add(Keys.NumWorkers)
   214→		for w := 0; w < Keys.NumWorkers; w++ {
   215→			go func() {
   216→				defer wg.Done()
   217→				for item := range work {
   218→					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
   219→					if err != nil {
   220→						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
   221→						atomic.AddInt32(&errs, 1)
   222→						continue
   223→					}
   224→					if len(checkpoints) > 0 {
   225→						results <- hostResult{
   226→							checkpoints: checkpoints,
   227→							hostname:    item.host,
   228→							files:       files,
   229→							dir:         item.dir,
   230→						}
   231→					}
   232→				}
   233→			}()
   234→		}
   235→
   236→		go func() {
   237→			for _, hostEntry := range hostEntries {
   238→				if !hostEntry.IsDir() {
   239→					continue
   240→				}
   241→				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
   242→				work <- struct {
   243→					dir, host string
   244→				}{dir: dir, host: hostEntry.Name()}
   245→			}
   246→			close(work)
   247→			wg.Wait()
   248→			close(results)
   249→		}()
   250→
   251→		// Open streaming writer and write each host's checkpoint files as a row group
   252→		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
   253→		writer, err := newParquetArchiveWriter(parquetFile)
   254→		if err != nil {
   255→			// Drain results channel to unblock workers
   256→			for range results {
   257→			}
   258→			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
   259→		}
   260→
   261→		type deleteItem struct {
   262→			dir   string
   263→			files []string
   264→		}
   265→		var toDelete []deleteItem
   266→		writeErr := error(nil)
   267→
   268→		for r := range results {
   269→			if writeErr == nil {
   270→				// Stream each checkpoint file directly to parquet rows.
   271→				// Each checkpoint is processed and discarded before the next.
   272→				for _, cf := range r.checkpoints {
   273→					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
   274→						writeErr = err
   275→						break
   276→					}
   277→				}
   278→				// Flush once per host to keep row group count within parquet limits.
   279→				if writeErr == nil {
   280→					if err := writer.FlushRowGroup(); err != nil {
   281→						writeErr = err
   282→					}
   283→				}
   284→			}
   285→			// Always track files for deletion (even if write failed, we still drain)
   286→			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
   287→		}
   288→
   289→		if err := writer.Close(); err != nil && writeErr == nil {
   290→			writeErr = err
   291→		}
   292→
   293→		if errs > 0 {
   294→			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
   295→		}
   296→
   297→		if writer.count == 0 {
   298→			// No data written — remove empty file
   299→			os.Remove(parquetFile)
   300→			continue
   301→		}
   302→
   303→		if writeErr != nil {
   304→			os.Remove(parquetFile)
   305→			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
   306→		}
   307→
   308→		// Delete archived checkpoint files
   309→		for _, item := range toDelete {
   310→			for _, file := range item.files {
   311→				filename := filepath.Join(item.dir, file)
   312→				if err := os.Remove(filename); err != nil {
   313→					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
   314→				} else {
   315→					totalFiles++
   316→				}
   317→			}
   318→		}
   319→

// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}


   319→
   320→		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
   321→			writer.count, totalFiles, cluster, parquetFile)
   322→	}
   323→
   324→	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
   325→	return totalFiles, nil
   326→}
   327→

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}


Now I have the full picture. Let me apply the changes:


Edit

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {

	totalFiles := 0
	var clusterErrors []string

	for _, clusterEntry := range clusterEntries {


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go has been updated successfully.

	totalFiles := 0
	var clusterErrors []string

	for _, clusterEntry := range clusterEntries {

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}


		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}


		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0
	var clusterErrors []string

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}

+      cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())

+      clusterErrors = append(clusterErrors, cluster)


		// Drain results channel to unblock workers
		for range results {
		}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)

		// Drain results channel to unblock workers
		for range results {
		}
			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue


<tool_use_error>String to replace not found in file.
String: 		// Drain results channel to unblock workers
		for range results {
		}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)</tool_use_error>

Error: String to replace not found in file.
String: 		// Drain results channel to unblock workers
		for range results {
		}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)


   252→		}()
   253→
   254→		// Open streaming writer and write each host's checkpoint files as a row group
   255→		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
   256→		writer, err := newParquetArchiveWriter(parquetFile)
   257→		if err != nil {
   258→			// Drain results channel to unblock workers
   259→			for range results {
   260→			}
   261→			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
   262→		}
   263→
   264→		type deleteItem struct {
   265→			dir   string
   266→			files []string

		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string


return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)

			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue


			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0
	var clusterErrors []string

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}

-      return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)

+      cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())


		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if errs > 0 {
			cclog.Errorf("[METRICSTORE]> %d errors reading checkpoints for cluster %s", errs, cluster)
			clusterErrors = append(clusterErrors, cluster)
			os.Remove(parquetFile)
			continue
		}


		if errs > 0 {
			cclog.Errorf("[METRICSTORE]> %d errors reading checkpoints for cluster %s", errs, cluster)
			clusterErrors = append(clusterErrors, cluster)
			os.Remove(parquetFile)
			continue
		}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0
	var clusterErrors []string

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}

-      return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)

+      cclog.Errorf("[METRICSTORE]> %d errors reading checkpoints for cluster %s", errs, cluster)


		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			cclog.Errorf("[METRICSTORE]> error writing parquet archive for cluster %s: %s", cluster, writeErr.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}


		if writeErr != nil {
			os.Remove(parquetFile)
			cclog.Errorf("[METRICSTORE]> error writing parquet archive for cluster %s: %s", cluster, writeErr.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0
	var clusterErrors []string

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			cclog.Errorf("[METRICSTORE]> %d errors reading checkpoints for cluster %s", errs, cluster)
			clusterErrors = append(clusterErrors, cluster)
			os.Remove(parquetFile)
			continue
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}

-      return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)

+      cclog.Errorf("[METRICSTORE]> error writing parquet archive for cluster %s: %s", cluster, writeErr.Error())


Now add the summary error after the loop:


	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)

	if len(clusterErrors) > 0 {
		return totalFiles, fmt.Errorf("archiving failed for clusters: %s", strings.Join(clusterErrors, ", "))
	}

	return totalFiles, nil


	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)

	if len(clusterErrors) > 0 {
		return totalFiles, fmt.Errorf("archiving failed for clusters: %s", strings.Join(clusterErrors, ", "))
	}

	return totalFiles, nil

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0
	var clusterErrors []string

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			cclog.Errorf("[METRICSTORE]> %d errors reading checkpoints for cluster %s", errs, cluster)
			clusterErrors = append(clusterErrors, cluster)
			os.Remove(parquetFile)
			continue
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			cclog.Errorf("[METRICSTORE]> error writing parquet archive for cluster %s: %s", cluster, writeErr.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}

+    return totalFiles, fmt.Errorf("archiving failed for clusters: %s", strings.Join(clusterErrors, ", "))


Check if `strings` is imported:


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"context"
    10→	"errors"
    11→	"fmt"
    12→	"os"
    13→	"path/filepath"
    14→	"sync"
    15→	"sync/atomic"
    16→	"time"
    17→
    18→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    19→)
    20→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)


Need to add `strings` import:


	"os"
	"path/filepath"
	"strings"
	"sync"


	"os"
	"path/filepath"
	"strings"
	"sync"

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0
	var clusterErrors []string

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			cclog.Errorf("[METRICSTORE]> %d errors reading checkpoints for cluster %s", errs, cluster)
			clusterErrors = append(clusterErrors, cluster)
			os.Remove(parquetFile)
			continue
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			cclog.Errorf("[METRICSTORE]> error writing parquet archive for cluster %s: %s", cluster, writeErr.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)

	if len(clusterErrors) > 0 {
		return totalFiles, fmt.Errorf("archiving failed for clusters: %s", strings.Join(clusterErrors, ", "))
	}

	return totalFiles, nil
}

"fmt"
"os"
"path/filepath"
+  "strings"
"sync"
"sync/atomic"
"time"


Bash

Build metricstore package


(Bash completed with no output)


Run metricstore tests


ok  	github.com/ClusterCockpit/cc-backend/pkg/metricstore	0.552s


Build and tests pass. Here's what was changed in `pkg/metricstore/archive.go`:

1. Added `var clusterErrors []string` before the cluster loop
2. **Line ~193** (`os.ReadDir` for hosts): log + append + `continue` instead of `return`
3. **Line ~261** (`newParquetArchiveWriter` failure): log + append + `continue` instead of `return`
4. **Line ~297** (worker `errs > 0`): log + append + `os.Remove` + `continue` instead of `return`
5. **Line ~309** (`writeErr != nil`): log + append + `continue` instead of `return`
6. After the loop: return a summary error listing all failed clusters if any errors occurred