     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"bufio"
    10→	"encoding/binary"
    11→	"encoding/json"
    12→	"fmt"
    13→	"os"
    14→	"path/filepath"
    15→	"sort"
    16→
    17→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    18→	pq "github.com/parquet-go/parquet-go"
    19→)
    20→
    21→// ParquetMetricRow is the long-format schema for archived metric data.
    22→// One row per (host, metric, scope, scope_id, timestamp) data point.
    23→// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
    24→type ParquetMetricRow struct {
    25→	Cluster   string  `parquet:"cluster"`
    26→	Hostname  string  `parquet:"hostname"`
    27→	Metric    string  `parquet:"metric"`
    28→	Scope     string  `parquet:"scope"`
    29→	ScopeID   string  `parquet:"scope_id"`
    30→	Timestamp int64   `parquet:"timestamp"`
    31→	Frequency int64   `parquet:"frequency"`
    32→	Value     float32 `parquet:"value"`
    33→}
    34→
    35→// flattenCheckpointFile recursively converts a CheckpointFile tree into Parquet rows.
    36→// The scope path is built from the hierarchy: host level is "node", then child names
    37→// map to scope/scope_id (e.g., "socket0" → scope="socket", scope_id="0").
    38→func flattenCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string, rows []ParquetMetricRow) []ParquetMetricRow {
    39→	for metricName, cm := range cf.Metrics {
    40→		ts := cm.Start
    41→		for _, v := range cm.Data {
    42→			if !v.IsNaN() {
    43→				rows = append(rows, ParquetMetricRow{
    44→					Cluster:   cluster,
    45→					Hostname:  hostname,
    46→					Metric:    metricName,
    47→					Scope:     scope,
    48→					ScopeID:   scopeID,
    49→					Timestamp: ts,
    50→					Frequency: cm.Frequency,
    51→					Value:     float32(v),
    52→				})
    53→			}
    54→			ts += cm.Frequency
    55→		}
    56→	}
    57→
    58→	for childName, childCf := range cf.Children {
    59→		childScope, childScopeID := parseScopeFromName(childName)
    60→		rows = flattenCheckpointFile(childCf, cluster, hostname, childScope, childScopeID, rows)
    61→	}
    62→
    63→	return rows
    64→}
    65→
    66→// parseScopeFromName infers scope and scope_id from a child level name.
    67→// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
    68→// "a0" (accelerator) → ("accelerator", "0").
    69→// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
    70→func parseScopeFromName(name string) (string, string) {
    71→	prefixes := []struct {
    72→		prefix string
    73→		scope  string
    74→	}{
    75→		{"socket", "socket"},
    76→		{"memoryDomain", "memoryDomain"},
    77→		{"core", "core"},
    78→		{"hwthread", "hwthread"},
    79→		{"cpu", "hwthread"},
    80→		{"accelerator", "accelerator"},
    81→	}
    82→
    83→	for _, p := range prefixes {
    84→		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
    85→			id := name[len(p.prefix):]
    86→			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
    87→				return p.scope, id
    88→			}
    89→		}
    90→	}
    91→
    92→	return name, ""
    93→}
    94→
    95→// parquetArchiveWriter supports incremental writes to a Parquet file.
    96→// Each call to WriteHostRows writes one row group (typically one host's data),
    97→// avoiding accumulation of all rows in memory.
    98→type parquetArchiveWriter struct {
    99→	writer *pq.GenericWriter[ParquetMetricRow]
   100→	bw     *bufio.Writer
   101→	f      *os.File
   102→	count  int
   103→}
   104→
   105→// newParquetArchiveWriter creates a streaming Parquet writer with Zstd compression.
   106→func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
   107→	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
   108→		return nil, fmt.Errorf("creating archive directory: %w", err)
   109→	}
   110→
   111→	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   112→	if err != nil {
   113→		return nil, fmt.Errorf("creating parquet file: %w", err)
   114→	}
   115→
   116→	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer
   117→
   118→	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
   119→		pq.Compression(&pq.Zstd),
   120→	)
   121→
   122→	return &parquetArchiveWriter{writer: writer, bw: bw, f: f}, nil
   123→}
   124→
   125→// WriteHostRows sorts rows by (metric, timestamp) in-place, writes them,
   126→// and flushes to create a separate row group.
   127→func (w *parquetArchiveWriter) WriteHostRows(rows []ParquetMetricRow) error {
   128→	sort.Slice(rows, func(i, j int) bool {
   129→		if rows[i].Metric != rows[j].Metric {
   130→			return rows[i].Metric < rows[j].Metric
   131→		}
   132→		return rows[i].Timestamp < rows[j].Timestamp
   133→	})
   134→
   135→	if _, err := w.writer.Write(rows); err != nil {
   136→		return fmt.Errorf("writing parquet rows: %w", err)
   137→	}
   138→
   139→	if err := w.writer.Flush(); err != nil {
   140→		return fmt.Errorf("flushing parquet row group: %w", err)
   141→	}
   142→
   143→	w.count += len(rows)
   144→	return nil
   145→}
   146→
   147→// Close finalises the Parquet file (footer, buffered I/O, file handle).
   148→func (w *parquetArchiveWriter) Close() error {
   149→	if err := w.writer.Close(); err != nil {
   150→		w.f.Close()
   151→		return fmt.Errorf("closing parquet writer: %w", err)
   152→	}
   153→
   154→	if err := w.bw.Flush(); err != nil {
   155→		w.f.Close()
   156→		return fmt.Errorf("flushing parquet file: %w", err)
   157→	}
   158→
   159→	return w.f.Close()
   160→}
   161→
   162→// sortParquetRows sorts rows by (metric, timestamp) in-place.
   163→func sortParquetRows(rows []ParquetMetricRow) {
   164→	sort.Slice(rows, func(i, j int) bool {
   165→		if rows[i].Metric != rows[j].Metric {
   166→			return rows[i].Metric < rows[j].Metric
   167→		}
   168→		return rows[i].Timestamp < rows[j].Timestamp
   169→	})
   170→}
   171→
   172→// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
   173→// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
   174→// before converting it to Parquet format.
   175→func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
   176→	f, err := os.Open(filename)
   177→	if err != nil {
   178→		return nil, err
   179→	}
   180→	defer f.Close()
   181→
   182→	ext := filepath.Ext(filename)
   183→	switch ext {
   184→	case ".json":
   185→		cf := &CheckpointFile{}
   186→		br := bufio.NewReader(f)
   187→		if err := json.NewDecoder(br).Decode(cf); err != nil {
   188→			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
   189→		}
   190→		return cf, nil
   191→
   192→	case ".bin":
   193→		br := bufio.NewReader(f)
   194→		var magic uint32
   195→		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
   196→			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
   197→		}
   198→		if magic != snapFileMagic {
   199→			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
   200→		}
   201→		var fileFrom, fileTo int64
   202→		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
   203→			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
   204→		}
   205→		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
   206→			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
   207→		}
   208→		cf, err := readBinaryLevel(br)
   209→		if err != nil {
   210→			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
   211→		}
   212→		cf.From = fileFrom
   213→		cf.To = fileTo
   214→		return cf, nil
   215→
   216→	default:
   217→		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
   218→	}
   219→}
   220→
   221→// estimateRowCount estimates the number of Parquet rows a CheckpointFile will produce.
   222→// Used for pre-allocating the rows slice to avoid repeated append doubling.
   223→func estimateRowCount(cf *CheckpointFile) int {
   224→	n := 0
   225→	for _, cm := range cf.Metrics {
   226→		n += len(cm.Data)
   227→	}
   228→	for _, child := range cf.Children {
   229→		n += estimateRowCount(child)
   230→	}
   231→	return n
   232→}
   233→
   234→// archiveCheckpointsToParquet reads checkpoint files for a host directory,
   235→// converts them to Parquet rows. Returns the rows and filenames that were processed.
   236→func archiveCheckpointsToParquet(dir, cluster, host string, from int64) ([]ParquetMetricRow, []string, error) {
   237→	entries, err := os.ReadDir(dir)
   238→	if err != nil {
   239→		return nil, nil, err
   240→	}
   241→
   242→	files, err := findFiles(entries, from, false)
   243→	if err != nil {
   244→		return nil, nil, err
   245→	}
   246→
   247→	if len(files) == 0 {
   248→		return nil, nil, nil
   249→	}
   250→
   251→	// First pass: load checkpoints and estimate total rows for pre-allocation.
   252→	type loaded struct {
   253→		cf       *CheckpointFile
   254→		filename string
   255→	}
   256→	var checkpoints []loaded
   257→	totalEstimate := 0
   258→
   259→	for _, checkpoint := range files {
   260→		filename := filepath.Join(dir, checkpoint)
   261→		cf, err := loadCheckpointFileFromDisk(filename)
   262→		if err != nil {
   263→			cclog.Warnf("[METRICSTORE]> skipping unreadable checkpoint %s: %v", filename, err)
   264→			continue
   265→		}
   266→		totalEstimate += estimateRowCount(cf)
   267→		checkpoints = append(checkpoints, loaded{cf: cf, filename: checkpoint})
   268→	}
   269→
   270→	if len(checkpoints) == 0 {
   271→		return nil, nil, nil
   272→	}
   273→
   274→	rows := make([]ParquetMetricRow, 0, totalEstimate)
   275→	processedFiles := make([]string, 0, len(checkpoints))
   276→
   277→	for _, cp := range checkpoints {
   278→		rows = flattenCheckpointFile(cp.cf, cluster, host, "node", "", rows)
   279→		processedFiles = append(processedFiles, cp.filename)
   280→	}
   281→
   282→	return rows, processedFiles, nil
   283→}
   284→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"bufio"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	pq "github.com/parquet-go/parquet-go"
)

// ParquetMetricRow is the long-format schema for archived metric data.
// One row per (host, metric, scope, scope_id, timestamp) data point.
// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
type ParquetMetricRow struct {
	Cluster   string  `parquet:"cluster"`
	Hostname  string  `parquet:"hostname"`
	Metric    string  `parquet:"metric"`
	Scope     string  `parquet:"scope"`
	ScopeID   string  `parquet:"scope_id"`
	Timestamp int64   `parquet:"timestamp"`
	Frequency int64   `parquet:"frequency"`
	Value     float32 `parquet:"value"`
}

// flattenCheckpointFile recursively converts a CheckpointFile tree into Parquet rows.
// The scope path is built from the hierarchy: host level is "node", then child names
// map to scope/scope_id (e.g., "socket0" → scope="socket", scope_id="0").
func flattenCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string, rows []ParquetMetricRow) []ParquetMetricRow {
	for metricName, cm := range cf.Metrics {
		ts := cm.Start
		for _, v := range cm.Data {
			if !v.IsNaN() {
				rows = append(rows, ParquetMetricRow{
					Cluster:   cluster,
					Hostname:  hostname,
					Metric:    metricName,
					Scope:     scope,
					ScopeID:   scopeID,
					Timestamp: ts,
					Frequency: cm.Frequency,
					Value:     float32(v),
				})
			}
			ts += cm.Frequency
		}
	}

	for childName, childCf := range cf.Children {
		childScope, childScopeID := parseScopeFromName(childName)
		rows = flattenCheckpointFile(childCf, cluster, hostname, childScope, childScopeID, rows)
	}

	return rows
}

// parseScopeFromName infers scope and scope_id from a child level name.
// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
// "a0" (accelerator) → ("accelerator", "0").
// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
func parseScopeFromName(name string) (string, string) {
	prefixes := []struct {
		prefix string
		scope  string
	}{
		{"socket", "socket"},
		{"memoryDomain", "memoryDomain"},
		{"core", "core"},
		{"hwthread", "hwthread"},
		{"cpu", "hwthread"},
		{"accelerator", "accelerator"},
	}

	for _, p := range prefixes {
		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
			id := name[len(p.prefix):]
			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
				return p.scope, id
			}
		}
	}

	return name, ""
}

// parquetArchiveWriter supports incremental writes to a Parquet file.
// Each call to WriteHostRows writes one row group (typically one host's data),
// avoiding accumulation of all rows in memory.
type parquetArchiveWriter struct {
	writer *pq.GenericWriter[ParquetMetricRow]
	bw     *bufio.Writer
	f      *os.File
	count  int
}

// newParquetArchiveWriter creates a streaming Parquet writer with Zstd compression.
func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
		return nil, fmt.Errorf("creating archive directory: %w", err)
	}

	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return nil, fmt.Errorf("creating parquet file: %w", err)
	}

	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer

	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
		pq.Compression(&pq.Zstd),
	)

	return &parquetArchiveWriter{writer: writer, bw: bw, f: f}, nil
}

// WriteHostRows sorts rows by (metric, timestamp) in-place, writes them,
// and flushes to create a separate row group.
func (w *parquetArchiveWriter) WriteHostRows(rows []ParquetMetricRow) error {
	sort.Slice(rows, func(i, j int) bool {
		if rows[i].Metric != rows[j].Metric {
			return rows[i].Metric < rows[j].Metric
		}
		return rows[i].Timestamp < rows[j].Timestamp
	})

	if _, err := w.writer.Write(rows); err != nil {
		return fmt.Errorf("writing parquet rows: %w", err)
	}

	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}

	w.count += len(rows)
	return nil
}

// Close finalises the Parquet file (footer, buffered I/O, file handle).
func (w *parquetArchiveWriter) Close() error {
	if err := w.writer.Close(); err != nil {
		w.f.Close()
		return fmt.Errorf("closing parquet writer: %w", err)
	}

	if err := w.bw.Flush(); err != nil {
		w.f.Close()
		return fmt.Errorf("flushing parquet file: %w", err)
	}

	return w.f.Close()
}

// sortParquetRows sorts rows by (metric, timestamp) in-place.
func sortParquetRows(rows []ParquetMetricRow) {
	sort.Slice(rows, func(i, j int) bool {
		if rows[i].Metric != rows[j].Metric {
			return rows[i].Metric < rows[j].Metric
		}
		return rows[i].Timestamp < rows[j].Timestamp
	})
}

// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
// before converting it to Parquet format.
func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ext := filepath.Ext(filename)
	switch ext {
	case ".json":
		cf := &CheckpointFile{}
		br := bufio.NewReader(f)
		if err := json.NewDecoder(br).Decode(cf); err != nil {
			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
		}
		return cf, nil

	case ".bin":
		br := bufio.NewReader(f)
		var magic uint32
		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
		}
		if magic != snapFileMagic {
			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
		}
		var fileFrom, fileTo int64
		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
		}
		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
		}
		cf, err := readBinaryLevel(br)
		if err != nil {
			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
		}
		cf.From = fileFrom
		cf.To = fileTo
		return cf, nil

	default:
		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
	}
}

// estimateRowCount estimates the number of Parquet rows a CheckpointFile will produce.
// Used for pre-allocating the rows slice to avoid repeated append doubling.
func estimateRowCount(cf *CheckpointFile) int {
	n := 0
	for _, cm := range cf.Metrics {
		n += len(cm.Data)
	}
	for _, child := range cf.Children {
		n += estimateRowCount(child)
	}
	return n
}

// archiveCheckpointsToParquet reads checkpoint files for a host directory,
// converts them to Parquet rows. Returns the rows and filenames that were processed.
func archiveCheckpointsToParquet(dir, cluster, host string, from int64) ([]ParquetMetricRow, []string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, nil, err
	}

	files, err := findFiles(entries, from, false)
	if err != nil {
		return nil, nil, err
	}

	if len(files) == 0 {
		return nil, nil, nil
	}

	// First pass: load checkpoints and estimate total rows for pre-allocation.
	type loaded struct {
		cf       *CheckpointFile
		filename string
	}
	var checkpoints []loaded
	totalEstimate := 0

	for _, checkpoint := range files {
		filename := filepath.Join(dir, checkpoint)
		cf, err := loadCheckpointFileFromDisk(filename)
		if err != nil {
			cclog.Warnf("[METRICSTORE]> skipping unreadable checkpoint %s: %v", filename, err)
			continue
		}
		totalEstimate += estimateRowCount(cf)
		checkpoints = append(checkpoints, loaded{cf: cf, filename: checkpoint})
	}

	if len(checkpoints) == 0 {
		return nil, nil, nil
	}

	rows := make([]ParquetMetricRow, 0, totalEstimate)
	processedFiles := make([]string, 0, len(checkpoints))

	for _, cp := range checkpoints {
		rows = flattenCheckpointFile(cp.cf, cluster, host, "node", "", rows)
		processedFiles = append(processedFiles, cp.filename)
	}

	return rows, processedFiles, nil
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"bufio"
    10→	"encoding/binary"
    11→	"encoding/json"
    12→	"fmt"
    13→	"os"
    14→	"path/filepath"
    15→
    16→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    17→	pq "github.com/parquet-go/parquet-go"
    18→)
    19→
    20→// ParquetMetricRow is the long-format schema for archived metric data.
    21→// One row per (host, metric, scope, scope_id, timestamp) data point.
    22→// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
    23→type ParquetMetricRow struct {
    24→	Cluster   string  `parquet:"cluster"`
    25→	Hostname  string  `parquet:"hostname"`
    26→	Metric    string  `parquet:"metric"`
    27→	Scope     string  `parquet:"scope"`
    28→	ScopeID   string  `parquet:"scope_id"`
    29→	Timestamp int64   `parquet:"timestamp"`
    30→	Frequency int64   `parquet:"frequency"`
    31→	Value     float32 `parquet:"value"`
    32→}
    33→
    34→// flattenCheckpointFile recursively converts a CheckpointFile tree into Parquet rows via a channel.
    35→// The scope path is built from the hierarchy: host level is "node", then child names
    36→// map to scope/scope_id (e.g., "socket0" → scope="socket", scope_id="0").
    37→func flattenCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string, rowChan chan<- *ParquetMetricRow) {
    38→	for metricName, cm := range cf.Metrics {
    39→		ts := cm.Start
    40→		for _, v := range cm.Data {
    41→			if !v.IsNaN() {
    42→				rowChan <- &ParquetMetricRow{
    43→					Cluster:   cluster,
    44→					Hostname:  hostname,
    45→					Metric:    metricName,
    46→					Scope:     scope,
    47→					ScopeID:   scopeID,
    48→					Timestamp: ts,
    49→					Frequency: cm.Frequency,
    50→					Value:     float32(v),
    51→				}
    52→			}
    53→			ts += cm.Frequency
    54→		}
    55→	}
    56→
    57→	for childName, childCf := range cf.Children {
    58→		childScope, childScopeID := parseScopeFromName(childName)
    59→		flattenCheckpointFile(childCf, cluster, hostname, childScope, childScopeID, rowChan)
    60→	}
    61→}
    62→
    63→// parseScopeFromName infers scope and scope_id from a child level name.
    64→// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
    65→// "a0" (accelerator) → ("accelerator", "0").
    66→// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
    67→func parseScopeFromName(name string) (string, string) {
    68→	prefixes := []struct {
    69→		prefix string
    70→		scope  string
    71→	}{
    72→		{"socket", "socket"},
    73→		{"memoryDomain", "memoryDomain"},
    74→		{"core", "core"},
    75→		{"hwthread", "hwthread"},
    76→		{"cpu", "hwthread"},
    77→		{"accelerator", "accelerator"},
    78→	}
    79→
    80→	for _, p := range prefixes {
    81→		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
    82→			id := name[len(p.prefix):]
    83→			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
    84→				return p.scope, id
    85→			}
    86→		}
    87→	}
    88→
    89→	return name, ""
    90→}
    91→
    92→// writeParquetArchiveStream writes rows from a channel to a Parquet file with Zstd compression in batches.
    93→func writeParquetArchiveStream(filename string, rowChan <-chan *ParquetMetricRow) (int, error) {
    94→	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
    95→		go func() {
    96→			for range rowChan {
    97→			}
    98→		}()
    99→		return 0, fmt.Errorf("creating archive directory: %w", err)
   100→	}
   101→
   102→	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   103→	if err != nil {
   104→		go func() {
   105→			for range rowChan {
   106→			}
   107→		}()
   108→		return 0, fmt.Errorf("creating parquet file: %w", err)
   109→	}
   110→	defer f.Close()
   111→
   112→	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer
   113→
   114→	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
   115→		pq.Compression(&pq.Zstd),
   116→		pq.SortingWriterConfig(pq.SortingColumns(
   117→			pq.Ascending("cluster"),
   118→			pq.Ascending("hostname"),
   119→			pq.Ascending("metric"),
   120→			pq.Ascending("timestamp"),
   121→		)),
   122→	)
   123→
   124→	batchSize := 4096
   125→	batch := make([]ParquetMetricRow, 0, batchSize)
   126→	rowCount := 0
   127→	var writeErr error
   128→
   129→	for rowPtr := range rowChan {
   130→		if writeErr != nil {
   131→			continue // Drain the channel to prevent worker deadlock
   132→		}
   133→		batch = append(batch, *rowPtr)
   134→		if len(batch) >= batchSize {
   135→			if _, err := writer.Write(batch); err != nil {
   136→				writeErr = fmt.Errorf("writing parquet batch: %w", err)
   137→			}
   138→			rowCount += len(batch)
   139→			batch = batch[:0]
   140→		}
   141→	}
   142→
   143→	if writeErr != nil {
   144→		return rowCount, writeErr
   145→	}
   146→
   147→	if len(batch) > 0 {
   148→		if _, err := writer.Write(batch); err != nil {
   149→			return rowCount, fmt.Errorf("writing remaining parquet batch: %w", err)
   150→		}
   151→		rowCount += len(batch)
   152→	}
   153→
   154→	if err := writer.Close(); err != nil {
   155→		return rowCount, fmt.Errorf("closing parquet writer: %w", err)
   156→	}
   157→
   158→	if err := bw.Flush(); err != nil {
   159→		return rowCount, fmt.Errorf("flushing parquet file: %w", err)
   160→	}
   161→
   162→	return rowCount, nil
   163→}
   164→
   165→// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
   166→// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
   167→// before converting it to Parquet format.
   168→func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
   169→	f, err := os.Open(filename)
   170→	if err != nil {
   171→		return nil, err
   172→	}
   173→	defer f.Close()
   174→
   175→	ext := filepath.Ext(filename)
   176→	switch ext {
   177→	case ".json":
   178→		cf := &CheckpointFile{}
   179→		br := bufio.NewReader(f)
   180→		if err := json.NewDecoder(br).Decode(cf); err != nil {
   181→			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
   182→		}
   183→		return cf, nil
   184→
   185→	case ".bin":
   186→		br := bufio.NewReader(f)
   187→		var magic uint32
   188→		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
   189→			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
   190→		}
   191→		if magic != snapFileMagic {
   192→			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
   193→		}
   194→		var fileFrom, fileTo int64
   195→		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
   196→			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
   197→		}
   198→		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
   199→			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
   200→		}
   201→		cf, err := readBinaryLevel(br)
   202→		if err != nil {
   203→			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
   204→		}
   205→		cf.From = fileFrom
   206→		cf.To = fileTo
   207→		return cf, nil
   208→
   209→	default:
   210→		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
   211→	}
   212→}
   213→
   214→// archiveCheckpointsToParquet reads checkpoint files for a host directory,
   215→// converts them to Parquet rows. Returns the filenames that were processed.
   216→func archiveCheckpointsToParquet(dir, cluster, host string, from int64, rowChan chan<- *ParquetMetricRow) ([]string, error) {
   217→	entries, err := os.ReadDir(dir)
   218→	if err != nil {
   219→		return nil, err
   220→	}
   221→
   222→	files, err := findFiles(entries, from, false)
   223→	if err != nil {
   224→		return nil, err
   225→	}
   226→
   227→	if len(files) == 0 {
   228→		return nil, nil
   229→	}
   230→
   231→	for _, checkpoint := range files {
   232→		filename := filepath.Join(dir, checkpoint)
   233→		cf, err := loadCheckpointFileFromDisk(filename)
   234→		if err != nil {
   235→			cclog.Warnf("[METRICSTORE]> skipping unreadable checkpoint %s: %v", filename, err)
   236→			continue
   237→		}
   238→
   239→		flattenCheckpointFile(cf, cluster, host, "node", "", rowChan)
   240→	}
   241→
   242→	return files, nil
   243→}
   244→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"bufio"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	pq "github.com/parquet-go/parquet-go"
)

// ParquetMetricRow is the long-format schema for archived metric data.
// One row per (host, metric, scope, scope_id, timestamp) data point.
// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
type ParquetMetricRow struct {
	Cluster   string  `parquet:"cluster"`
	Hostname  string  `parquet:"hostname"`
	Metric    string  `parquet:"metric"`
	Scope     string  `parquet:"scope"`
	ScopeID   string  `parquet:"scope_id"`
	Timestamp int64   `parquet:"timestamp"`
	Frequency int64   `parquet:"frequency"`
	Value     float32 `parquet:"value"`
}

// flattenCheckpointFile recursively converts a CheckpointFile tree into Parquet rows via a channel.
// The scope path is built from the hierarchy: host level is "node", then child names
// map to scope/scope_id (e.g., "socket0" → scope="socket", scope_id="0").
func flattenCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string, rowChan chan<- *ParquetMetricRow) {
	for metricName, cm := range cf.Metrics {
		ts := cm.Start
		for _, v := range cm.Data {
			if !v.IsNaN() {
				rowChan <- &ParquetMetricRow{
					Cluster:   cluster,
					Hostname:  hostname,
					Metric:    metricName,
					Scope:     scope,
					ScopeID:   scopeID,
					Timestamp: ts,
					Frequency: cm.Frequency,
					Value:     float32(v),
				}
			}
			ts += cm.Frequency
		}
	}

	for childName, childCf := range cf.Children {
		childScope, childScopeID := parseScopeFromName(childName)
		flattenCheckpointFile(childCf, cluster, hostname, childScope, childScopeID, rowChan)
	}
}

// parseScopeFromName infers scope and scope_id from a child level name.
// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
// "a0" (accelerator) → ("accelerator", "0").
// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
func parseScopeFromName(name string) (string, string) {
	prefixes := []struct {
		prefix string
		scope  string
	}{
		{"socket", "socket"},
		{"memoryDomain", "memoryDomain"},
		{"core", "core"},
		{"hwthread", "hwthread"},
		{"cpu", "hwthread"},
		{"accelerator", "accelerator"},
	}

	for _, p := range prefixes {
		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
			id := name[len(p.prefix):]
			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
				return p.scope, id
			}
		}
	}

	return name, ""
}

// writeParquetArchiveStream writes rows from a channel to a Parquet file with Zstd compression in batches.
func writeParquetArchiveStream(filename string, rowChan <-chan *ParquetMetricRow) (int, error) {
	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
		go func() {
			for range rowChan {
			}
		}()
		return 0, fmt.Errorf("creating archive directory: %w", err)
	}

	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		go func() {
			for range rowChan {
			}
		}()
		return 0, fmt.Errorf("creating parquet file: %w", err)
	}
	defer f.Close()

	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer

	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
		pq.Compression(&pq.Zstd),
		pq.SortingWriterConfig(pq.SortingColumns(
			pq.Ascending("cluster"),
			pq.Ascending("hostname"),
			pq.Ascending("metric"),
			pq.Ascending("timestamp"),
		)),
	)

	batchSize := 4096
	batch := make([]ParquetMetricRow, 0, batchSize)
	rowCount := 0
	var writeErr error

	for rowPtr := range rowChan {
		if writeErr != nil {
			continue // Drain the channel to prevent worker deadlock
		}
		batch = append(batch, *rowPtr)
		if len(batch) >= batchSize {
			if _, err := writer.Write(batch); err != nil {
				writeErr = fmt.Errorf("writing parquet batch: %w", err)
			}
			rowCount += len(batch)
			batch = batch[:0]
		}
	}

	if writeErr != nil {
		return rowCount, writeErr
	}

	if len(batch) > 0 {
		if _, err := writer.Write(batch); err != nil {
			return rowCount, fmt.Errorf("writing remaining parquet batch: %w", err)
		}
		rowCount += len(batch)
	}

	if err := writer.Close(); err != nil {
		return rowCount, fmt.Errorf("closing parquet writer: %w", err)
	}

	if err := bw.Flush(); err != nil {
		return rowCount, fmt.Errorf("flushing parquet file: %w", err)
	}

	return rowCount, nil
}

// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
// before converting it to Parquet format.
func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ext := filepath.Ext(filename)
	switch ext {
	case ".json":
		cf := &CheckpointFile{}
		br := bufio.NewReader(f)
		if err := json.NewDecoder(br).Decode(cf); err != nil {
			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
		}
		return cf, nil

	case ".bin":
		br := bufio.NewReader(f)
		var magic uint32
		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
		}
		if magic != snapFileMagic {
			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
		}
		var fileFrom, fileTo int64
		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
		}
		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
		}
		cf, err := readBinaryLevel(br)
		if err != nil {
			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
		}
		cf.From = fileFrom
		cf.To = fileTo
		return cf, nil

	default:
		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
	}
}

// archiveCheckpointsToParquet reads checkpoint files for a host directory,
// converts them to Parquet rows. Returns the filenames that were processed.
func archiveCheckpointsToParquet(dir, cluster, host string, from int64, rowChan chan<- *ParquetMetricRow) ([]string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	files, err := findFiles(entries, from, false)
	if err != nil {
		return nil, err
	}

	if len(files) == 0 {
		return nil, nil
	}

	for _, checkpoint := range files {
		filename := filepath.Join(dir, checkpoint)
		cf, err := loadCheckpointFileFromDisk(filename)
		if err != nil {
			cclog.Warnf("[METRICSTORE]> skipping unreadable checkpoint %s: %v", filename, err)
			continue
		}

		flattenCheckpointFile(cf, cluster, host, "node", "", rowChan)
	}

	return files, nil
}


Two different approaches to the same problem. Here's how they compare:

## Architecture: Batch-per-host (ours) vs Channel-streaming (tmp)

### `flattenCheckpointFile`

| | Ours (hotfix) | tmp |
|---|---|---|
| **Signature** | `(..., rows []ParquetMetricRow) []ParquetMetricRow` | `(..., rowChan chan<- *ParquetMetricRow)` |
| **Mechanism** | Appends to a slice, returns it | Sends `*ParquetMetricRow` pointers to a channel |
| **Allocation** | Struct values in a contiguous slice | One heap-allocated pointer per row |

### Writer

| | Ours (`parquetArchiveWriter`) | tmp (`writeParquetArchiveStream`) |
|---|---|---|
| **API** | Struct with `WriteHostRows()` + `Close()` | Single function consuming `<-chan *ParquetMetricRow` |
| **SortingWriterConfig** | Removed | **Still present** (lines 116-121) |
| **Row groups** | One per host (explicit `Flush()`) | One big row group (only `Close()` at end) |
| **Sorting** | Pre-sorted in-place per host via `sort.Slice` | Delegated to `SortingWriterConfig` internally |
| **Batching** | Entire host's rows at once | Fixed 4096-row batches from channel |

### `archiveCheckpointsToParquet`

| | Ours | tmp |
|---|---|---|
| **Returns** | `([]ParquetMetricRow, []string, error)` — rows + files | `([]string, error)` — files only |
| **Pre-allocation** | `estimateRowCount()` → `make([]ParquetMetricRow, 0, n)` | No pre-allocation (rows go through channel) |
| **Two-pass loading** | Yes: load all checkpoints, estimate, then flatten | No: single pass, streams directly to channel |

## Key Differences

**1. The tmp version still has the `SortingWriterConfig` problem.** It batches 4096 rows at a time into `writer.Write()`, but `SortingWriterConfig` buffers ALL written rows internally until `writer.Close()`. So despite the channel streaming, the sorting writer still accumulates every row in memory — the memory fix is incomplete.

**2. Per-row heap allocation in tmp.** Each row is `&ParquetMetricRow{...}` sent as a pointer through the channel. For millions of rows this creates significant GC pressure vs our approach of contiguous slice values.

**3. Our version creates proper row groups.** Each `WriteHostRows()` + `Flush()` creates a separate Parquet row group per host, enabling row-group skipping on read. The tmp version produces one monolithic row group.

**4. Error handling on early failure.** The tmp version spawns goroutines to drain the channel if `MkdirAll` or `OpenFile` fails (lines 95-98, 104-107). Our version avoids this by opening the writer before starting workers (in `archive.go`).

**5. Our version has `estimateRowCount` + pre-allocation** to avoid append doubling. The tmp version doesn't need this (no slice accumulation) but trades it for per-row channel send overhead.

## Bottom line

The tmp version streams row-by-row through a channel (good for decoupling producer/consumer) but **defeats its own purpose** by keeping `SortingWriterConfig`, which buffers everything internally anyway. Our version removes `SortingWriterConfig`, pre-sorts per host, and writes proper row groups — actually solving the memory problem.


Add an Info logmessage in archiveCheckpoints that archving started and provide timing information how long it took.


   169→// archiveCheckpoints archives checkpoint files to Parquet format.
   170→// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
   171→// Each host's rows are written as a separate row group to avoid accumulating
   172→// all data in memory at once.
   173→func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
   174→	clusterEntries, err := os.ReadDir(checkpointsDir)
   175→	if err != nil {
   176→		return 0, err
   177→	}
   178→
   179→	totalFiles := 0
   180→
   181→	for _, clusterEntry := range clusterEntries {
   182→		if !clusterEntry.IsDir() {
   183→			continue

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Each host's rows are written as a separate row group to avoid accumulating
// all data in memory at once.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"context"
    10→	"errors"
    11→	"fmt"
    12→	"os"
    13→	"path/filepath"
    14→	"sync"
    15→	"sync/atomic"
    16→	"time"
    17→
    18→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    19→)
    20→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)


Good, `time` is already imported.


Edit

func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0


The file /Users/jan/prg/CC/cc-backend/pkg/metricstore/archive.go has been updated successfully.

func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {

		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir            string
		cluster, host  string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Each host's rows are written as a separate row group to avoid accumulating
// all data in memory at once.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Stream per-host rows to parquet writer via worker pool
		type hostResult struct {
			rows  []ParquetMetricRow
			files []string // checkpoint filenames to delete after successful write
			dir   string   // checkpoint directory for this host
		}

		results := make(chan hostResult, len(hostEntries))
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					rows, files, err := archiveCheckpointsToParquet(item.dir, cluster, item.host, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(rows) > 0 {
						results <- hostResult{rows: rows, files: files, dir: item.dir}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's rows as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				sortParquetRows(r.rows)
				if err := writer.WriteHostRows(r.rows); err != nil {
					writeErr = err
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	return totalFiles, nil
}

+  cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")


Now read the end of the function to add the timing log before the return:


   275→		if errs > 0 {
   276→			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
   277→		}
   278→
   279→		if writer.count == 0 {
   280→			// No data written — remove empty file
   281→			os.Remove(parquetFile)
   282→			continue
   283→		}
   284→

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}


   295→					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
   296→				} else {
   297→					totalFiles++
   298→				}
   299→			}
   300→		}
   301→
   302→		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
   303→			writer.count, totalFiles, cluster, parquetFile)
   304→	}

					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}


   304→	}
   305→
   306→	return totalFiles, nil
   307→}
   308→

	}

	return totalFiles, nil
}


	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}


	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {

		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir            string
		cluster, host  string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Each host's rows are written as a separate row group to avoid accumulating
// all data in memory at once.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Stream per-host rows to parquet writer via worker pool
		type hostResult struct {
			rows  []ParquetMetricRow
			files []string // checkpoint filenames to delete after successful write
			dir   string   // checkpoint directory for this host
		}

		results := make(chan hostResult, len(hostEntries))
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					rows, files, err := archiveCheckpointsToParquet(item.dir, cluster, item.host, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(rows) > 0 {
						results <- hostResult{rows: rows, files: files, dir: item.dir}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's rows as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				sortParquetRows(r.rows)
				if err := writer.WriteHostRows(r.rows); err != nil {
					writeErr = err
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	return totalFiles, nil
}

+  cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)


Bash

Run go vet


(Bash completed with no output)


Added an info log at the start (`start archiving checkpoints to parquet`) and a completion log with elapsed time and file count (`archiving checkpoints completed in 1.234s (42 files)`).