Implement the following plan:

# Fix: Parquet Row Group Limit in Metric Store Archiving

## Context

The cleanup archiver creates one parquet file per cluster and calls `writer.Flush()` after every checkpoint file, creating one row group per checkpoint. The parquet-go library hard-limits row groups to 32767 per file. Large clusters like "fritz" with many hosts and multiple checkpoint files per host exceed this limit, causing: `flushing parquet row group: the limit of 32767 row groups has been reached`

## Root Cause

In `WriteCheckpointFile()` (`parquetArchive.go:115`), `w.writer.Flush()` is called after **every** checkpoint file. For a cluster with e.g. 1000 hosts × 40 checkpoint files = 40,000 flushes → 40,000 row groups → exceeds 32767.

## Fix

Move the `Flush()` out of `WriteCheckpointFile` and into the caller, flushing **once per host** instead of once per checkpoint file. This gives at most one row group per host (e.g. ~3000 for a large cluster), well within the limit.

This also improves compression — all checkpoint data for a single host is now in one row group with highly repetitive column values.

### File 1: `pkg/metricstore/parquetArchive.go`

1. **Remove `Flush()` from `WriteCheckpointFile`** (lines 115-117) — keep only the batch-drain logic
2. **Add `FlushRowGroup()` method** — wraps `w.writer.Flush()` for the caller to invoke explicitly

### File 2: `pkg/metricstore/archive.go`

3. **Call `FlushRowGroup()` once per host** in the `for r := range results` loop (after all checkpoint files for that host are written), instead of relying on per-checkpoint flushes:

```go
for r := range results {
    if writeErr == nil {
        for _, cf := range r.checkpoints {
            if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
                writeErr = err
                break
            }
        }
        if writeErr == nil {
            if err := writer.FlushRowGroup(); err != nil {
                writeErr = err
            }
        }
    }
    toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
}
```

## Verification

- `go build ./pkg/metricstore/...` — compiles
- `go test ./pkg/metricstore/...` — existing tests pass
- Manually verify: for a cluster with N hosts, the parquet file should contain N row groups (not N × checkpoints-per-host)

If you need specific details from before exiting plan mode (like exact code snippets, error messages, or content you generated), read the full transcript at: /Users/jan/.claude/projects/-Users-jan-prg-ClusterCockpit-cc-backend/94e2d90e-5772-4423-a4fb-75c7caa1e1c4.jsonl


   100→// WriteCheckpointFile streams a CheckpointFile tree directly to Parquet rows,
   101→// writing metrics in sorted order without materializing all rows in memory.
   102→// Produces one row group per call (typically one host's data).
   103→func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {
   104→	w.writeLevel(cf, cluster, hostname, scope, scopeID)
   105→
   106→	// Flush remaining batch
   107→	if len(w.batch) > 0 {
   108→		if _, err := w.writer.Write(w.batch); err != nil {
   109→			return fmt.Errorf("writing parquet rows: %w", err)
   110→		}
   111→		w.count += len(w.batch)
   112→		w.batch = w.batch[:0]
   113→	}
   114→
   115→	if err := w.writer.Flush(); err != nil {
   116→		return fmt.Errorf("flushing parquet row group: %w", err)
   117→	}
   118→
   119→	return nil
   120→}
   121→
   122→// writeLevel recursively writes metrics from a CheckpointFile level.
   123→// Metric names and child names are sorted for deterministic, compression-friendly output.
   124→func (w *parquetArchiveWriter) writeLevel(cf *CheckpointFile, cluster, hostname, scope, scopeID string) {
   125→	// Sort metric names for deterministic order
   126→	metricNames := make([]string, 0, len(cf.Metrics))
   127→	for name := range cf.Metrics {
   128→		metricNames = append(metricNames, name)
   129→	}

// WriteCheckpointFile streams a CheckpointFile tree directly to Parquet rows,
// writing metrics in sorted order without materializing all rows in memory.
// Produces one row group per call (typically one host's data).
func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {
	w.writeLevel(cf, cluster, hostname, scope, scopeID)

	// Flush remaining batch
	if len(w.batch) > 0 {
		if _, err := w.writer.Write(w.batch); err != nil {
			return fmt.Errorf("writing parquet rows: %w", err)
		}
		w.count += len(w.batch)
		w.batch = w.batch[:0]
	}

	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}

	return nil
}

// writeLevel recursively writes metrics from a CheckpointFile level.
// Metric names and child names are sorted for deterministic, compression-friendly output.
func (w *parquetArchiveWriter) writeLevel(cf *CheckpointFile, cluster, hostname, scope, scopeID string) {
	// Sort metric names for deterministic order
	metricNames := make([]string, 0, len(cf.Metrics))
	for name := range cf.Metrics {
		metricNames = append(metricNames, name)
	}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"context"
    10→	"errors"
    11→	"fmt"
    12→	"os"
    13→	"path/filepath"
    14→	"sync"
    15→	"sync/atomic"
    16→	"time"
    17→
    18→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    19→)
    20→
    21→// Worker for either Archiving or Deleting files
    22→
    23→func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
    24→	if Keys.Cleanup.Mode == "archive" {
    25→		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
    26→		// Run as Archiver
    27→		cleanUpWorker(wg, ctx,
    28→			Keys.RetentionInMemory,
    29→			"archiving",
    30→			Keys.Cleanup.RootDir,
    31→			false,
    32→		)
    33→	} else {
    34→		// Run as Deleter
    35→		cleanUpWorker(wg, ctx,
    36→			Keys.RetentionInMemory,
    37→			"deleting",
    38→			"",
    39→			true,
    40→		)
    41→	}
    42→}
    43→
    44→// cleanUpWorker takes simple values to configure what it does
    45→func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
    46→	wg.Go(func() {
    47→		d, err := time.ParseDuration(interval)
    48→		if err != nil {
    49→			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
    50→		}
    51→		if d <= 0 {
    52→			return
    53→		}
    54→
    55→		ticker := time.NewTicker(d)
    56→		defer ticker.Stop()
    57→
    58→		for {
    59→			select {
    60→			case <-ctx.Done():
    61→				return
    62→			case <-ticker.C:
    63→				t := time.Now().Add(-d)
    64→				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))
    65→
    66→				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)
    67→
    68→				if err != nil {
    69→					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
    70→				} else {
    71→					if delete {
    72→						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
    73→					} else {
    74→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
    75→					}
    76→				}
    77→			}
    78→		}
    79→	})
    80→}
    81→
    82→var ErrNoNewArchiveData error = errors.New("all data already archived")
    83→
    84→// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
    85→// When archiving, consolidates all hosts per cluster into a single Parquet file.
    86→func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
    87→	if deleteInstead {
    88→		return deleteCheckpoints(checkpointsDir, from)
    89→	}
    90→
    91→	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
    92→}
    93→
    94→// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
    95→func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
    96→	entries1, err := os.ReadDir(checkpointsDir)
    97→	if err != nil {
    98→		return 0, err
    99→	}
   100→
   101→	type workItem struct {
   102→		dir           string
   103→		cluster, host string
   104→	}
   105→
   106→	var wg sync.WaitGroup
   107→	n, errs := int32(0), int32(0)
   108→	work := make(chan workItem, Keys.NumWorkers)
   109→
   110→	wg.Add(Keys.NumWorkers)
   111→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   112→		go func() {
   113→			defer wg.Done()
   114→			for item := range work {
   115→				entries, err := os.ReadDir(item.dir)
   116→				if err != nil {
   117→					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
   118→					atomic.AddInt32(&errs, 1)
   119→					continue
   120→				}
   121→
   122→				files, err := findFiles(entries, from, false)
   123→				if err != nil {
   124→					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
   125→					atomic.AddInt32(&errs, 1)
   126→					continue
   127→				}
   128→
   129→				for _, checkpoint := range files {
   130→					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
   131→						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
   132→						atomic.AddInt32(&errs, 1)
   133→					} else {
   134→						atomic.AddInt32(&n, 1)
   135→					}
   136→				}
   137→			}
   138→		}()
   139→	}
   140→
   141→	for _, de1 := range entries1 {
   142→		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
   143→		if e != nil {
   144→			err = e
   145→			continue
   146→		}
   147→
   148→		for _, de2 := range entries2 {
   149→			work <- workItem{
   150→				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
   151→				cluster: de1.Name(),
   152→				host:    de2.Name(),
   153→			}
   154→		}
   155→	}
   156→
   157→	close(work)
   158→	wg.Wait()
   159→
   160→	if err != nil {
   161→		return int(n), err
   162→	}
   163→	if errs > 0 {
   164→		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
   165→	}
   166→	return int(n), nil
   167→}
   168→
   169→// archiveCheckpoints archives checkpoint files to Parquet format.
   170→// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
   171→// Workers load checkpoint files from disk and send CheckpointFile trees on a
   172→// back-pressured channel. The main thread streams each tree directly to Parquet
   173→// rows without materializing all rows in memory.
   174→func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
   175→	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
   176→	startTime := time.Now()
   177→
   178→	clusterEntries, err := os.ReadDir(checkpointsDir)
   179→	if err != nil {
   180→		return 0, err
   181→	}
   182→
   183→	totalFiles := 0
   184→
   185→	for _, clusterEntry := range clusterEntries {
   186→		if !clusterEntry.IsDir() {
   187→			continue
   188→		}
   189→
   190→		cluster := clusterEntry.Name()
   191→		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
   192→		if err != nil {
   193→			return totalFiles, err
   194→		}
   195→
   196→		// Workers load checkpoint files from disk; main thread writes to parquet.
   197→		type hostResult struct {
   198→			checkpoints []*CheckpointFile
   199→			hostname    string
   200→			files       []string // checkpoint filenames to delete after successful write
   201→			dir         string   // checkpoint directory for this host
   202→		}
   203→
   204→		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
   205→		results := make(chan hostResult, 2)
   206→		work := make(chan struct {
   207→			dir, host string
   208→		}, Keys.NumWorkers)
   209→
   210→		var wg sync.WaitGroup
   211→		errs := int32(0)
   212→
   213→		wg.Add(Keys.NumWorkers)
   214→		for w := 0; w < Keys.NumWorkers; w++ {
   215→			go func() {
   216→				defer wg.Done()
   217→				for item := range work {
   218→					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
   219→					if err != nil {
   220→						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
   221→						atomic.AddInt32(&errs, 1)
   222→						continue
   223→					}
   224→					if len(checkpoints) > 0 {
   225→						results <- hostResult{
   226→							checkpoints: checkpoints,
   227→							hostname:    item.host,
   228→							files:       files,
   229→							dir:         item.dir,
   230→						}
   231→					}
   232→				}
   233→			}()
   234→		}
   235→
   236→		go func() {
   237→			for _, hostEntry := range hostEntries {
   238→				if !hostEntry.IsDir() {
   239→					continue
   240→				}
   241→				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
   242→				work <- struct {
   243→					dir, host string
   244→				}{dir: dir, host: hostEntry.Name()}
   245→			}
   246→			close(work)
   247→			wg.Wait()
   248→			close(results)
   249→		}()
   250→
   251→		// Open streaming writer and write each host's checkpoint files as a row group
   252→		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
   253→		writer, err := newParquetArchiveWriter(parquetFile)
   254→		if err != nil {
   255→			// Drain results channel to unblock workers
   256→			for range results {
   257→			}
   258→			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
   259→		}
   260→
   261→		type deleteItem struct {
   262→			dir   string
   263→			files []string
   264→		}
   265→		var toDelete []deleteItem
   266→		writeErr := error(nil)
   267→
   268→		for r := range results {
   269→			if writeErr == nil {
   270→				// Stream each checkpoint file directly to parquet rows.
   271→				// Each checkpoint is processed and discarded before the next.
   272→				for _, cf := range r.checkpoints {
   273→					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
   274→						writeErr = err
   275→						break
   276→					}
   277→				}
   278→			}
   279→			// Always track files for deletion (even if write failed, we still drain)
   280→			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
   281→		}
   282→
   283→		if err := writer.Close(); err != nil && writeErr == nil {
   284→			writeErr = err
   285→		}
   286→
   287→		if errs > 0 {
   288→			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
   289→		}
   290→
   291→		if writer.count == 0 {
   292→			// No data written — remove empty file
   293→			os.Remove(parquetFile)
   294→			continue
   295→		}
   296→
   297→		if writeErr != nil {
   298→			os.Remove(parquetFile)
   299→			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
   300→		}
   301→
   302→		// Delete archived checkpoint files
   303→		for _, item := range toDelete {
   304→			for _, file := range item.files {
   305→				filename := filepath.Join(item.dir, file)
   306→				if err := os.Remove(filename); err != nil {
   307→					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
   308→				} else {
   309→					totalFiles++
   310→				}
   311→			}
   312→		}
   313→
   314→		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
   315→			writer.count, totalFiles, cluster, parquetFile)
   316→	}
   317→
   318→	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
   319→	return totalFiles, nil
   320→}
   321→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}


Edit

	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}

	return nil
}

	return nil
}

// FlushRowGroup flushes the current row group to the Parquet file.
// Should be called once per host after all checkpoint files for that host are written.
func (w *parquetArchiveWriter) FlushRowGroup() error {
	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}
	return nil
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/parquetArchive.go has been updated successfully.

	return nil
}

// FlushRowGroup flushes the current row group to the Parquet file.
// Should be called once per host after all checkpoint files for that host are written.
func (w *parquetArchiveWriter) FlushRowGroup() error {
	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}
	return nil
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"bufio"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"

	pq "github.com/parquet-go/parquet-go"
)

// ParquetMetricRow is the long-format schema for archived metric data.
// One row per (host, metric, scope, scope_id, timestamp) data point.
// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
type ParquetMetricRow struct {
	Cluster   string  `parquet:"cluster"`
	Hostname  string  `parquet:"hostname"`
	Metric    string  `parquet:"metric"`
	Scope     string  `parquet:"scope"`
	ScopeID   string  `parquet:"scope_id"`
	Timestamp int64   `parquet:"timestamp"`
	Frequency int64   `parquet:"frequency"`
	Value     float32 `parquet:"value"`
}

// parseScopeFromName infers scope and scope_id from a child level name.
// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
// "a0" (accelerator) → ("accelerator", "0").
// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
func parseScopeFromName(name string) (string, string) {
	prefixes := []struct {
		prefix string
		scope  string
	}{
		{"socket", "socket"},
		{"memoryDomain", "memoryDomain"},
		{"core", "core"},
		{"hwthread", "hwthread"},
		{"cpu", "hwthread"},
		{"accelerator", "accelerator"},
	}

	for _, p := range prefixes {
		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
			id := name[len(p.prefix):]
			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
				return p.scope, id
			}
		}
	}

	return name, ""
}

// parquetArchiveWriter supports incremental writes to a Parquet file.
// Uses streaming writes to avoid accumulating all rows in memory.
type parquetArchiveWriter struct {
	writer *pq.GenericWriter[ParquetMetricRow]
	bw     *bufio.Writer
	f      *os.File
	batch  []ParquetMetricRow // reusable batch buffer
	count  int
}

const parquetBatchSize = 1024

// newParquetArchiveWriter creates a streaming Parquet writer with Zstd compression.
func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
		return nil, fmt.Errorf("creating archive directory: %w", err)
	}

	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return nil, fmt.Errorf("creating parquet file: %w", err)
	}

	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer

	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
		pq.Compression(&pq.Zstd),
	)

	return &parquetArchiveWriter{
		writer: writer,
		bw:     bw,
		f:      f,
		batch:  make([]ParquetMetricRow, 0, parquetBatchSize),
	}, nil
}

// WriteCheckpointFile streams a CheckpointFile tree directly to Parquet rows,
// writing metrics in sorted order without materializing all rows in memory.
// Produces one row group per call (typically one host's data).
func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {
	w.writeLevel(cf, cluster, hostname, scope, scopeID)

	// Flush remaining batch
	if len(w.batch) > 0 {
		if _, err := w.writer.Write(w.batch); err != nil {
			return fmt.Errorf("writing parquet rows: %w", err)
		}
		w.count += len(w.batch)
		w.batch = w.batch[:0]
	}

	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}

	return nil
}

// writeLevel recursively writes metrics from a CheckpointFile level.
// Metric names and child names are sorted for deterministic, compression-friendly output.
func (w *parquetArchiveWriter) writeLevel(cf *CheckpointFile, cluster, hostname, scope, scopeID string) {
	// Sort metric names for deterministic order
	metricNames := make([]string, 0, len(cf.Metrics))
	for name := range cf.Metrics {
		metricNames = append(metricNames, name)
	}
	sort.Strings(metricNames)

	for _, metricName := range metricNames {
		cm := cf.Metrics[metricName]
		ts := cm.Start
		for _, v := range cm.Data {
			if !v.IsNaN() {
				w.batch = append(w.batch, ParquetMetricRow{
					Cluster:   cluster,
					Hostname:  hostname,
					Metric:    metricName,
					Scope:     scope,
					ScopeID:   scopeID,
					Timestamp: ts,
					Frequency: cm.Frequency,
					Value:     float32(v),
				})

				if len(w.batch) >= parquetBatchSize {
					w.writer.Write(w.batch)
					w.count += len(w.batch)
					w.batch = w.batch[:0]
				}
			}
			ts += cm.Frequency
		}
	}

	// Sort child names for deterministic order
	childNames := make([]string, 0, len(cf.Children))
	for name := range cf.Children {
		childNames = append(childNames, name)
	}
	sort.Strings(childNames)

	for _, childName := range childNames {
		childScope, childScopeID := parseScopeFromName(childName)
		w.writeLevel(cf.Children[childName], cluster, hostname, childScope, childScopeID)
	}
}

// Close finalises the Parquet file (footer, buffered I/O, file handle).
func (w *parquetArchiveWriter) Close() error {
	if err := w.writer.Close(); err != nil {
		w.f.Close()
		return fmt.Errorf("closing parquet writer: %w", err)
	}

	if err := w.bw.Flush(); err != nil {
		w.f.Close()
		return fmt.Errorf("flushing parquet file: %w", err)
	}

	return w.f.Close()
}

// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
// before converting it to Parquet format.
func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ext := filepath.Ext(filename)
	switch ext {
	case ".json":
		cf := &CheckpointFile{}
		br := bufio.NewReader(f)
		if err := json.NewDecoder(br).Decode(cf); err != nil {
			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
		}
		return cf, nil

	case ".bin":
		br := bufio.NewReader(f)
		var magic uint32
		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
		}
		if magic != snapFileMagic {
			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
		}
		var fileFrom, fileTo int64
		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
		}
		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
		}
		cf, err := readBinaryLevel(br)
		if err != nil {
			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
		}
		cf.From = fileFrom
		cf.To = fileTo
		return cf, nil

	default:
		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
	}
}

// loadCheckpointFiles reads checkpoint files for a host directory and returns
// the loaded CheckpointFiles and their filenames. Processes one file at a time
// to avoid holding all checkpoint data in memory simultaneously.
func loadCheckpointFiles(dir string, from int64) ([]*CheckpointFile, []string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, nil, err
	}

	files, err := findFiles(entries, from, false)
	if err != nil {
		return nil, nil, err
	}

	if len(files) == 0 {
		return nil, nil, nil
	}

	var checkpoints []*CheckpointFile
	var processedFiles []string

	for _, checkpoint := range files {
		filename := filepath.Join(dir, checkpoint)
		cf, err := loadCheckpointFileFromDisk(filename)
		if err != nil {
			continue
		}
		checkpoints = append(checkpoints, cf)
		processedFiles = append(processedFiles, checkpoint)
	}

	return checkpoints, processedFiles, nil
}

+// FlushRowGroup flushes the current row group to the Parquet file.

+// Should be called once per host after all checkpoint files for that host are written.

+func (w *parquetArchiveWriter) FlushRowGroup() error {


// Produces one row group per call (typically one host's data).

// Call FlushRowGroup() after writing all checkpoint files for a host.


// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"bufio"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"

	pq "github.com/parquet-go/parquet-go"
)

// ParquetMetricRow is the long-format schema for archived metric data.
// One row per (host, metric, scope, scope_id, timestamp) data point.
// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
type ParquetMetricRow struct {
	Cluster   string  `parquet:"cluster"`
	Hostname  string  `parquet:"hostname"`
	Metric    string  `parquet:"metric"`
	Scope     string  `parquet:"scope"`
	ScopeID   string  `parquet:"scope_id"`
	Timestamp int64   `parquet:"timestamp"`
	Frequency int64   `parquet:"frequency"`
	Value     float32 `parquet:"value"`
}

// parseScopeFromName infers scope and scope_id from a child level name.
// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
// "a0" (accelerator) → ("accelerator", "0").
// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
func parseScopeFromName(name string) (string, string) {
	prefixes := []struct {
		prefix string
		scope  string
	}{
		{"socket", "socket"},
		{"memoryDomain", "memoryDomain"},
		{"core", "core"},
		{"hwthread", "hwthread"},
		{"cpu", "hwthread"},
		{"accelerator", "accelerator"},
	}

	for _, p := range prefixes {
		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
			id := name[len(p.prefix):]
			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
				return p.scope, id
			}
		}
	}

	return name, ""
}

// parquetArchiveWriter supports incremental writes to a Parquet file.
// Uses streaming writes to avoid accumulating all rows in memory.
type parquetArchiveWriter struct {
	writer *pq.GenericWriter[ParquetMetricRow]
	bw     *bufio.Writer
	f      *os.File
	batch  []ParquetMetricRow // reusable batch buffer
	count  int
}

const parquetBatchSize = 1024

// newParquetArchiveWriter creates a streaming Parquet writer with Zstd compression.
func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
		return nil, fmt.Errorf("creating archive directory: %w", err)
	}

	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return nil, fmt.Errorf("creating parquet file: %w", err)
	}

	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer

	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
		pq.Compression(&pq.Zstd),
	)

	return &parquetArchiveWriter{
		writer: writer,
		bw:     bw,
		f:      f,
		batch:  make([]ParquetMetricRow, 0, parquetBatchSize),
	}, nil
}

// WriteCheckpointFile streams a CheckpointFile tree directly to Parquet rows,
// writing metrics in sorted order without materializing all rows in memory.
// Produces one row group per call (typically one host's data).
func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {
	w.writeLevel(cf, cluster, hostname, scope, scopeID)

	// Flush remaining batch
	if len(w.batch) > 0 {
		if _, err := w.writer.Write(w.batch); err != nil {
			return fmt.Errorf("writing parquet rows: %w", err)
		}
		w.count += len(w.batch)
		w.batch = w.batch[:0]
	}

	return nil
}

// FlushRowGroup flushes the current row group to the Parquet file.
// Should be called once per host after all checkpoint files for that host are written.
func (w *parquetArchiveWriter) FlushRowGroup() error {
	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}
	return nil
}

// writeLevel recursively writes metrics from a CheckpointFile level.
// Metric names and child names are sorted for deterministic, compression-friendly output.
func (w *parquetArchiveWriter) writeLevel(cf *CheckpointFile, cluster, hostname, scope, scopeID string) {
	// Sort metric names for deterministic order
	metricNames := make([]string, 0, len(cf.Metrics))
	for name := range cf.Metrics {
		metricNames = append(metricNames, name)
	}
	sort.Strings(metricNames)

	for _, metricName := range metricNames {
		cm := cf.Metrics[metricName]
		ts := cm.Start
		for _, v := range cm.Data {
			if !v.IsNaN() {
				w.batch = append(w.batch, ParquetMetricRow{
					Cluster:   cluster,
					Hostname:  hostname,
					Metric:    metricName,
					Scope:     scope,
					ScopeID:   scopeID,
					Timestamp: ts,
					Frequency: cm.Frequency,
					Value:     float32(v),
				})

				if len(w.batch) >= parquetBatchSize {
					w.writer.Write(w.batch)
					w.count += len(w.batch)
					w.batch = w.batch[:0]
				}
			}
			ts += cm.Frequency
		}
	}

	// Sort child names for deterministic order
	childNames := make([]string, 0, len(cf.Children))
	for name := range cf.Children {
		childNames = append(childNames, name)
	}
	sort.Strings(childNames)

	for _, childName := range childNames {
		childScope, childScopeID := parseScopeFromName(childName)
		w.writeLevel(cf.Children[childName], cluster, hostname, childScope, childScopeID)
	}
}

// Close finalises the Parquet file (footer, buffered I/O, file handle).
func (w *parquetArchiveWriter) Close() error {
	if err := w.writer.Close(); err != nil {
		w.f.Close()
		return fmt.Errorf("closing parquet writer: %w", err)
	}

	if err := w.bw.Flush(); err != nil {
		w.f.Close()
		return fmt.Errorf("flushing parquet file: %w", err)
	}

	return w.f.Close()
}

// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
// before converting it to Parquet format.
func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ext := filepath.Ext(filename)
	switch ext {
	case ".json":
		cf := &CheckpointFile{}
		br := bufio.NewReader(f)
		if err := json.NewDecoder(br).Decode(cf); err != nil {
			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
		}
		return cf, nil

	case ".bin":
		br := bufio.NewReader(f)
		var magic uint32
		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
		}
		if magic != snapFileMagic {
			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
		}
		var fileFrom, fileTo int64
		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
		}
		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
		}
		cf, err := readBinaryLevel(br)
		if err != nil {
			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
		}
		cf.From = fileFrom
		cf.To = fileTo
		return cf, nil

	default:
		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
	}
}

// loadCheckpointFiles reads checkpoint files for a host directory and returns
// the loaded CheckpointFiles and their filenames. Processes one file at a time
// to avoid holding all checkpoint data in memory simultaneously.
func loadCheckpointFiles(dir string, from int64) ([]*CheckpointFile, []string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, nil, err
	}

	files, err := findFiles(entries, from, false)
	if err != nil {
		return nil, nil, err
	}

	if len(files) == 0 {
		return nil, nil, nil
	}

	var checkpoints []*CheckpointFile
	var processedFiles []string

	for _, checkpoint := range files {
		filename := filepath.Join(dir, checkpoint)
		cf, err := loadCheckpointFileFromDisk(filename)
		if err != nil {
			continue
		}
		checkpoints = append(checkpoints, cf)
		processedFiles = append(processedFiles, checkpoint)
	}

	return checkpoints, processedFiles, nil
}

-// Produces one row group per call (typically one host's data).

+// Call FlushRowGroup() after writing all checkpoint files for a host.


			for _, cf := range r.checkpoints {
				if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
					writeErr = err
					break
				}
			}

			for _, cf := range r.checkpoints {
				if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
					writeErr = err
					break
				}
			}
			if writeErr == nil {
				if err := writer.FlushRowGroup(); err != nil {
					writeErr = err
				}
			}


<tool_use_error>String to replace not found in file.
String: 			for _, cf := range r.checkpoints {
				if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
					writeErr = err
					break
				}
			}</tool_use_error>

Error: String to replace not found in file.
String: 			for _, cf := range r.checkpoints {
				if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
					writeErr = err
					break
				}
			}


   268→		for r := range results {
   269→			if writeErr == nil {
   270→				// Stream each checkpoint file directly to parquet rows.
   271→				// Each checkpoint is processed and discarded before the next.
   272→				for _, cf := range r.checkpoints {
   273→					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
   274→						writeErr = err
   275→						break
   276→					}
   277→				}
   278→			}
   279→			// Always track files for deletion (even if write failed, we still drain)
   280→			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
   281→		}
   282→
   283→		if err := writer.Close(); err != nil && writeErr == nil {
   284→			writeErr = err
   285→		}
   286→
   287→		if errs > 0 {

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {


I see — the indentation uses tabs. Let me match exactly.


		if writeErr == nil {
			// Stream each checkpoint file directly to parquet rows.
			// Each checkpoint is processed and discarded before the next.
			for _, cf := range r.checkpoints {
				if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
					writeErr = err
					break
				}
			}
		}

		if writeErr == nil {
			// Stream each checkpoint file directly to parquet rows.
			// Each checkpoint is processed and discarded before the next.
			for _, cf := range r.checkpoints {
				if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
					writeErr = err
					break
				}
			}
			// Flush once per host to keep row group count within parquet limits.
			if writeErr == nil {
				if err := writer.FlushRowGroup(); err != nil {
					writeErr = err
				}
			}
		}


<tool_use_error>String to replace not found in file.
String: 		if writeErr == nil {
			// Stream each checkpoint file directly to parquet rows.
			// Each checkpoint is processed and discarded before the next.
			for _, cf := range r.checkpoints {
				if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
					writeErr = err
					break
				}
			}
		}</tool_use_error>

Error: String to replace not found in file.
String: 		if writeErr == nil {
			// Stream each checkpoint file directly to parquet rows.
			// Each checkpoint is processed and discarded before the next.
			for _, cf := range r.checkpoints {
				if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
					writeErr = err
					break
				}
			}
		}


Bash

cat -A /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go | head -282 | tail -15

Check exact whitespace characters in file


cat: illegal option -- A
usage: cat [-belnstuv] [file ...]


sed -n '268,281p' /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go | cat -v

Show whitespace chars in lines 268-281


		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}


The file uses tabs. Let me try a different approach.


hexdump -C /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go | grep -A2 "268\|269\|270" | head -5

Check if file uses tabs


00000270  6e 55 70 57 6f 72 6b 65  72 28 77 67 2c 20 63 74  |nUpWorker(wg, ct|
00000280  78 2c 0a 09 09 09 4b 65  79 73 2e 52 65 74 65 6e  |x,....Keys.Reten|
00000290  74 69 6f 6e 49 6e 4d 65  6d 6f 72 79 2c 0a 09 09  |tionInMemory,...|
--
00001270  20 74 69 6d 65 2e 4e 6f  77 28 29 0a 0a 09 63 6c  | time.Now()...cl|


It uses tabs. The Read tool output already showed them as spaces in the line-numbered view but they're actually tabs. Let me use a direct write approach.


   268→		for r := range results {
   269→			if writeErr == nil {
   270→				// Stream each checkpoint file directly to parquet rows.
   271→				// Each checkpoint is processed and discarded before the next.
   272→				for _, cf := range r.checkpoints {
   273→					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
   274→						writeErr = err
   275→						break
   276→					}
   277→				}
   278→			}
   279→			// Always track files for deletion (even if write failed, we still drain)
   280→			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
   281→		}


sed -n '277p' pkg/metricstore/archive.go | od -c | head -3

Check exact chars on line 277


0000000   \t  \t  \t  \t   }  \n
0000006


Tabs confirmed. Let me use sed to insert the flush lines after line 277.


sed -i '' '277a\
\t\t\t\t// Flush once per host to keep row group count within parquet limits.\
\t\t\t\tif writeErr == nil {\
\t\t\t\t\tif err := writer.FlushRowGroup(); err != nil {\
\t\t\t\t\t\twriteErr = err\
\t\t\t\t\t}\
\t\t\t\t}
' pkg/metricstore/archive.go

Insert FlushRowGroup call after line 277


(Bash completed with no output)


   268→		for r := range results {
   269→			if writeErr == nil {
   270→				// Stream each checkpoint file directly to parquet rows.
   271→				// Each checkpoint is processed and discarded before the next.
   272→				for _, cf := range r.checkpoints {
   273→					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
   274→						writeErr = err
   275→						break
   276→					}
   277→				}
   278→tttt// Flush once per host to keep row group count within parquet limits.
   279→ttttif writeErr == nil {
   280→tttttif err := writer.FlushRowGroup(); err != nil {
   281→ttttttwriteErr = err
   282→ttttt}
   283→tttt}
   284→			}
   285→			// Always track files for deletion (even if write failed, we still drain)
   286→			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
   287→		}

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
tttt// Flush once per host to keep row group count within parquet limits.
ttttif writeErr == nil {
tttttif err := writer.FlushRowGroup(); err != nil {
ttttttwriteErr = err
ttttt}
tttt}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}


The sed didn't expand \t to tabs, it used literal 't'. Let me fix this properly.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"context"
    10→	"errors"
    11→	"fmt"
    12→	"os"
    13→	"path/filepath"
    14→	"sync"
    15→	"sync/atomic"
    16→	"time"
    17→
    18→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    19→)
    20→
    21→// Worker for either Archiving or Deleting files
    22→
    23→func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
    24→	if Keys.Cleanup.Mode == "archive" {
    25→		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
    26→		// Run as Archiver
    27→		cleanUpWorker(wg, ctx,
    28→			Keys.RetentionInMemory,
    29→			"archiving",
    30→			Keys.Cleanup.RootDir,
    31→			false,
    32→		)
    33→	} else {
    34→		// Run as Deleter
    35→		cleanUpWorker(wg, ctx,
    36→			Keys.RetentionInMemory,
    37→			"deleting",
    38→			"",
    39→			true,
    40→		)
    41→	}
    42→}
    43→
    44→// cleanUpWorker takes simple values to configure what it does
    45→func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
    46→	wg.Go(func() {
    47→		d, err := time.ParseDuration(interval)
    48→		if err != nil {
    49→			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
    50→		}
    51→		if d <= 0 {
    52→			return
    53→		}
    54→
    55→		ticker := time.NewTicker(d)
    56→		defer ticker.Stop()
    57→
    58→		for {
    59→			select {
    60→			case <-ctx.Done():
    61→				return
    62→			case <-ticker.C:
    63→				t := time.Now().Add(-d)
    64→				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))
    65→
    66→				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)
    67→
    68→				if err != nil {
    69→					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
    70→				} else {
    71→					if delete {
    72→						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
    73→					} else {
    74→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
    75→					}
    76→				}
    77→			}
    78→		}
    79→	})
    80→}
    81→
    82→var ErrNoNewArchiveData error = errors.New("all data already archived")
    83→
    84→// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
    85→// When archiving, consolidates all hosts per cluster into a single Parquet file.
    86→func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
    87→	if deleteInstead {
    88→		return deleteCheckpoints(checkpointsDir, from)
    89→	}
    90→
    91→	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
    92→}
    93→
    94→// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
    95→func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
    96→	entries1, err := os.ReadDir(checkpointsDir)
    97→	if err != nil {
    98→		return 0, err
    99→	}
   100→
   101→	type workItem struct {
   102→		dir           string
   103→		cluster, host string
   104→	}
   105→
   106→	var wg sync.WaitGroup
   107→	n, errs := int32(0), int32(0)
   108→	work := make(chan workItem, Keys.NumWorkers)
   109→
   110→	wg.Add(Keys.NumWorkers)
   111→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   112→		go func() {
   113→			defer wg.Done()
   114→			for item := range work {
   115→				entries, err := os.ReadDir(item.dir)
   116→				if err != nil {
   117→					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
   118→					atomic.AddInt32(&errs, 1)
   119→					continue
   120→				}
   121→
   122→				files, err := findFiles(entries, from, false)
   123→				if err != nil {
   124→					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
   125→					atomic.AddInt32(&errs, 1)
   126→					continue
   127→				}
   128→
   129→				for _, checkpoint := range files {
   130→					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
   131→						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
   132→						atomic.AddInt32(&errs, 1)
   133→					} else {
   134→						atomic.AddInt32(&n, 1)
   135→					}
   136→				}
   137→			}
   138→		}()
   139→	}
   140→
   141→	for _, de1 := range entries1 {
   142→		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
   143→		if e != nil {
   144→			err = e
   145→			continue
   146→		}
   147→
   148→		for _, de2 := range entries2 {
   149→			work <- workItem{
   150→				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
   151→				cluster: de1.Name(),
   152→				host:    de2.Name(),
   153→			}
   154→		}
   155→	}
   156→
   157→	close(work)
   158→	wg.Wait()
   159→
   160→	if err != nil {
   161→		return int(n), err
   162→	}
   163→	if errs > 0 {
   164→		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
   165→	}
   166→	return int(n), nil
   167→}
   168→
   169→// archiveCheckpoints archives checkpoint files to Parquet format.
   170→// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
   171→// Workers load checkpoint files from disk and send CheckpointFile trees on a
   172→// back-pressured channel. The main thread streams each tree directly to Parquet
   173→// rows without materializing all rows in memory.
   174→func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
   175→	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
   176→	startTime := time.Now()
   177→
   178→	clusterEntries, err := os.ReadDir(checkpointsDir)
   179→	if err != nil {
   180→		return 0, err
   181→	}
   182→
   183→	totalFiles := 0
   184→
   185→	for _, clusterEntry := range clusterEntries {
   186→		if !clusterEntry.IsDir() {
   187→			continue
   188→		}
   189→
   190→		cluster := clusterEntry.Name()
   191→		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
   192→		if err != nil {
   193→			return totalFiles, err
   194→		}
   195→
   196→		// Workers load checkpoint files from disk; main thread writes to parquet.
   197→		type hostResult struct {
   198→			checkpoints []*CheckpointFile
   199→			hostname    string
   200→			files       []string // checkpoint filenames to delete after successful write
   201→			dir         string   // checkpoint directory for this host
   202→		}
   203→
   204→		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
   205→		results := make(chan hostResult, 2)
   206→		work := make(chan struct {
   207→			dir, host string
   208→		}, Keys.NumWorkers)
   209→
   210→		var wg sync.WaitGroup
   211→		errs := int32(0)
   212→
   213→		wg.Add(Keys.NumWorkers)
   214→		for w := 0; w < Keys.NumWorkers; w++ {
   215→			go func() {
   216→				defer wg.Done()
   217→				for item := range work {
   218→					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
   219→					if err != nil {
   220→						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
   221→						atomic.AddInt32(&errs, 1)
   222→						continue
   223→					}
   224→					if len(checkpoints) > 0 {
   225→						results <- hostResult{
   226→							checkpoints: checkpoints,
   227→							hostname:    item.host,
   228→							files:       files,
   229→							dir:         item.dir,
   230→						}
   231→					}
   232→				}
   233→			}()
   234→		}
   235→
   236→		go func() {
   237→			for _, hostEntry := range hostEntries {
   238→				if !hostEntry.IsDir() {
   239→					continue
   240→				}
   241→				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
   242→				work <- struct {
   243→					dir, host string
   244→				}{dir: dir, host: hostEntry.Name()}
   245→			}
   246→			close(work)
   247→			wg.Wait()
   248→			close(results)
   249→		}()
   250→
   251→		// Open streaming writer and write each host's checkpoint files as a row group
   252→		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
   253→		writer, err := newParquetArchiveWriter(parquetFile)
   254→		if err != nil {
   255→			// Drain results channel to unblock workers
   256→			for range results {
   257→			}
   258→			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
   259→		}
   260→
   261→		type deleteItem struct {
   262→			dir   string
   263→			files []string
   264→		}
   265→		var toDelete []deleteItem
   266→		writeErr := error(nil)
   267→
   268→		for r := range results {
   269→			if writeErr == nil {
   270→				// Stream each checkpoint file directly to parquet rows.
   271→				// Each checkpoint is processed and discarded before the next.
   272→				for _, cf := range r.checkpoints {
   273→					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
   274→						writeErr = err
   275→						break
   276→					}
   277→				}
   278→tttt// Flush once per host to keep row group count within parquet limits.
   279→ttttif writeErr == nil {
   280→tttttif err := writer.FlushRowGroup(); err != nil {
   281→ttttttwriteErr = err
   282→ttttt}
   283→tttt}
   284→			}
   285→			// Always track files for deletion (even if write failed, we still drain)
   286→			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
   287→		}
   288→
   289→		if err := writer.Close(); err != nil && writeErr == nil {
   290→			writeErr = err
   291→		}
   292→
   293→		if errs > 0 {
   294→			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
   295→		}
   296→
   297→		if writer.count == 0 {
   298→			// No data written — remove empty file
   299→			os.Remove(parquetFile)
   300→			continue
   301→		}
   302→
   303→		if writeErr != nil {
   304→			os.Remove(parquetFile)
   305→			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
   306→		}
   307→
   308→		// Delete archived checkpoint files
   309→		for _, item := range toDelete {
   310→			for _, file := range item.files {
   311→				filename := filepath.Join(item.dir, file)
   312→				if err := os.Remove(filename); err != nil {
   313→					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
   314→				} else {
   315→					totalFiles++
   316→				}
   317→			}
   318→		}
   319→
   320→		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
   321→			writer.count, totalFiles, cluster, parquetFile)
   322→	}
   323→
   324→	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
   325→	return totalFiles, nil
   326→}
   327→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
tttt// Flush once per host to keep row group count within parquet limits.
ttttif writeErr == nil {
tttttif err := writer.FlushRowGroup(); err != nil {
ttttttwriteErr = err
ttttt}
tttt}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}


I see the broken lines 278-283 with literal "t" chars. Let me fix the whole block.


Edit

			}
tttt// Flush once per host to keep row group count within parquet limits.
ttttif writeErr == nil {
tttttif err := writer.FlushRowGroup(); err != nil {
ttttttwriteErr = err
ttttt}
tttt}
		}

			}
			// Flush once per host to keep row group count within parquet limits.
			if writeErr == nil {
				if err := writer.FlushRowGroup(); err != nil {
					writeErr = err
				}
			}
		}


<tool_use_error>String to replace not found in file.
String: 			}
tttt// Flush once per host to keep row group count within parquet limits.
ttttif writeErr == nil {
tttttif err := writer.FlushRowGroup(); err != nil {
ttttttwriteErr = err
ttttt}
tttt}
		}</tool_use_error>

Error: String to replace not found in file.
String: 			}
tttt// Flush once per host to keep row group count within parquet limits.
ttttif writeErr == nil {
tttttif err := writer.FlushRowGroup(); err != nil {
ttttttwriteErr = err
ttttt}
tttt}
		}


Bash

sed -n '277,284p' /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go | od -c

Show exact bytes of the broken lines


0000000   \t  \t  \t  \t   }  \n   t   t   t   t   /   /       F   l   u
0000020    s   h       o   n   c   e       p   e   r       h   o   s   t
0000040        t   o       k   e   e   p       r   o   w       g   r   o
0000060    u   p       c   o   u   n   t       w   i   t   h   i   n
0000100    p   a   r   q   u   e   t       l   i   m   i   t   s   .  \n
0000120    t   t   t   t   i   f       w   r   i   t   e   E   r   r
0000140    =   =       n   i   l       {  \n   t   t   t   t   t   i   f
0000160        e   r   r       :   =       w   r   i   t   e   r   .   F
0000200    l   u   s   h   R   o   w   G   r   o   u   p   (   )   ;
0000220    e   r   r       !   =       n   i   l       {  \n   t   t   t
0000240    t   t   t   w   r   i   t   e   E   r   r       =       e   r
0000260    r  \n   t   t   t   t   t   }  \n   t   t   t   t   }  \n  \t
0000300   \t  \t   }  \n
0000304


The broken lines have literal 't' characters (not tabs). Let me just rewrite the file with the correct content.


Write

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go has been updated successfully.

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}

-tttt// Flush once per host to keep row group count within parquet limits.

-tttttif err := writer.FlushRowGroup(); err != nil {

+        // Flush once per host to keep row group count within parquet limits.

+          if err := writer.FlushRowGroup(); err != nil {

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
tttt// Flush once per host to keep row group count within parquet limits.
ttttif writeErr == nil {
tttttif err := writer.FlushRowGroup(); err != nil {
ttttttwriteErr = err
ttttt}
tttt}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}


Now let me verify both files compile and tests pass.


Bash

Build metricstore package


Run metricstore tests


ok  	github.com/ClusterCockpit/cc-backend/pkg/metricstore	0.492s