Edit

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"bufio"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	pq "github.com/parquet-go/parquet-go"
)

// ParquetMetricRow is the long-format schema for archived metric data.
// One row per (host, metric, scope, scope_id, timestamp) data point.
// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
type ParquetMetricRow struct {
	Cluster   string  `parquet:"cluster"`
	Hostname  string  `parquet:"hostname"`
	Metric    string  `parquet:"metric"`
	Scope     string  `parquet:"scope"`
	ScopeID   string  `parquet:"scope_id"`
	Timestamp int64   `parquet:"timestamp"`
	Frequency int64   `parquet:"frequency"`
	Value     float32 `parquet:"value"`
}

// flattenCheckpointFile recursively converts a CheckpointFile tree into Parquet rows.
// The scope path is built from the hierarchy: host level is "node", then child names
// map to scope/scope_id (e.g., "socket0" → scope="socket", scope_id="0").
func flattenCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string, rows []ParquetMetricRow) []ParquetMetricRow {
	for metricName, cm := range cf.Metrics {
		ts := cm.Start
		for _, v := range cm.Data {
			if !v.IsNaN() {
				rows = append(rows, ParquetMetricRow{
					Cluster:   cluster,
					Hostname:  hostname,
					Metric:    metricName,
					Scope:     scope,
					ScopeID:   scopeID,
					Timestamp: ts,
					Frequency: cm.Frequency,
					Value:     float32(v),
				})
			}
			ts += cm.Frequency
		}
	}

	for childName, childCf := range cf.Children {
		childScope, childScopeID := parseScopeFromName(childName)
		rows = flattenCheckpointFile(childCf, cluster, hostname, childScope, childScopeID, rows)
	}

	return rows
}

// parseScopeFromName infers scope and scope_id from a child level name.
// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
// "a0" (accelerator) → ("accelerator", "0").
// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
func parseScopeFromName(name string) (string, string) {
	prefixes := []struct {
		prefix string
		scope  string
	}{
		{"socket", "socket"},
		{"memoryDomain", "memoryDomain"},
		{"core", "core"},
		{"hwthread", "hwthread"},
		{"cpu", "hwthread"},
		{"accelerator", "accelerator"},
	}

	for _, p := range prefixes {
		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
			id := name[len(p.prefix):]
			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
				return p.scope, id
			}
		}
	}

	return name, ""
}

// parquetArchiveWriter supports incremental writes to a Parquet file.
// Each call to WriteHostRows writes one row group (typically one host's data),
// avoiding accumulation of all rows in memory.
type parquetArchiveWriter struct {
	writer *pq.GenericWriter[ParquetMetricRow]
	bw     *bufio.Writer
	f      *os.File
	count  int
}

// newParquetArchiveWriter creates a streaming Parquet writer with Zstd compression.
func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
		return nil, fmt.Errorf("creating archive directory: %w", err)
	}

	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return nil, fmt.Errorf("creating parquet file: %w", err)
	}

	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer

	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
		pq.Compression(&pq.Zstd),
	)

	return &parquetArchiveWriter{writer: writer, bw: bw, f: f}, nil
}

// WriteHostRows sorts rows by (metric, timestamp) in-place, writes them,
// and flushes to create a separate row group.
func (w *parquetArchiveWriter) WriteHostRows(rows []ParquetMetricRow) error {
	sort.Slice(rows, func(i, j int) bool {
		if rows[i].Metric != rows[j].Metric {
			return rows[i].Metric < rows[j].Metric
		}
		return rows[i].Timestamp < rows[j].Timestamp
	})

	if _, err := w.writer.Write(rows); err != nil {
		return fmt.Errorf("writing parquet rows: %w", err)
	}

	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}

	w.count += len(rows)
	return nil
}

// Close finalises the Parquet file (footer, buffered I/O, file handle).
func (w *parquetArchiveWriter) Close() error {
	if err := w.writer.Close(); err != nil {
		w.f.Close()
		return fmt.Errorf("closing parquet writer: %w", err)
	}

	if err := w.bw.Flush(); err != nil {
		w.f.Close()
		return fmt.Errorf("flushing parquet file: %w", err)
	}

	return w.f.Close()
}

// sortParquetRows sorts rows by (metric, timestamp) in-place.
func sortParquetRows(rows []ParquetMetricRow) {
	sort.Slice(rows, func(i, j int) bool {
		if rows[i].Metric != rows[j].Metric {
			return rows[i].Metric < rows[j].Metric
		}
		return rows[i].Timestamp < rows[j].Timestamp
	})
}

// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
// before converting it to Parquet format.
func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ext := filepath.Ext(filename)
	switch ext {
	case ".json":
		cf := &CheckpointFile{}
		br := bufio.NewReader(f)
		if err := json.NewDecoder(br).Decode(cf); err != nil {
			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
		}
		return cf, nil

	case ".bin":
		br := bufio.NewReader(f)
		var magic uint32
		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
		}
		if magic != snapFileMagic {
			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
		}
		var fileFrom, fileTo int64
		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
		}
		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
		}
		cf, err := readBinaryLevel(br)
		if err != nil {
			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
		}
		cf.From = fileFrom
		cf.To = fileTo
		return cf, nil

	default:
		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
	}
}

// estimateRowCount estimates the number of Parquet rows a CheckpointFile will produce.
// Used for pre-allocating the rows slice to avoid repeated append doubling.
func estimateRowCount(cf *CheckpointFile) int {
	n := 0
	for _, cm := range cf.Metrics {
		n += len(cm.Data)
	}
	for _, child := range cf.Children {
		n += estimateRowCount(child)
	}
	return n
}

// archiveCheckpointsToParquet reads checkpoint files for a host directory,
// converts them to Parquet rows. Returns the rows and filenames that were processed.
func archiveCheckpointsToParquet(dir, cluster, host string, from int64) ([]ParquetMetricRow, []string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, nil, err
	}

	files, err := findFiles(entries, from, false)
	if err != nil {
		return nil, nil, err
	}

	if len(files) == 0 {
		return nil, nil, nil
	}

	// First pass: load checkpoints and estimate total rows for pre-allocation.
	type loaded struct {
		cf       *CheckpointFile
		filename string
	}
	var checkpoints []loaded
	totalEstimate := 0

	for _, checkpoint := range files {
		filename := filepath.Join(dir, checkpoint)
		cf, err := loadCheckpointFileFromDisk(filename)
		if err != nil {
			cclog.Warnf("[METRICSTORE]> skipping unreadable checkpoint %s: %v", filename, err)
			continue
		}
		totalEstimate += estimateRowCount(cf)
		checkpoints = append(checkpoints, loaded{cf: cf, filename: checkpoint})
	}

	if len(checkpoints) == 0 {
		return nil, nil, nil
	}

	rows := make([]ParquetMetricRow, 0, totalEstimate)
	processedFiles := make([]string, 0, len(checkpoints))

	for _, cp := range checkpoints {
		rows = flattenCheckpointFile(cp.cf, cluster, host, "node", "", rows)
		processedFiles = append(processedFiles, cp.filename)
	}

	return rows, processedFiles, nil
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"bufio"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"

	pq "github.com/parquet-go/parquet-go"
)

// ParquetMetricRow is the long-format schema for archived metric data.
// One row per (host, metric, scope, scope_id, timestamp) data point.
// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
type ParquetMetricRow struct {
	Cluster   string  `parquet:"cluster"`
	Hostname  string  `parquet:"hostname"`
	Metric    string  `parquet:"metric"`
	Scope     string  `parquet:"scope"`
	ScopeID   string  `parquet:"scope_id"`
	Timestamp int64   `parquet:"timestamp"`
	Frequency int64   `parquet:"frequency"`
	Value     float32 `parquet:"value"`
}

// parseScopeFromName infers scope and scope_id from a child level name.
// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
// "a0" (accelerator) → ("accelerator", "0").
// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
func parseScopeFromName(name string) (string, string) {
	prefixes := []struct {
		prefix string
		scope  string
	}{
		{"socket", "socket"},
		{"memoryDomain", "memoryDomain"},
		{"core", "core"},
		{"hwthread", "hwthread"},
		{"cpu", "hwthread"},
		{"accelerator", "accelerator"},
	}

	for _, p := range prefixes {
		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
			id := name[len(p.prefix):]
			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
				return p.scope, id
			}
		}
	}

	return name, ""
}

// parquetArchiveWriter supports incremental writes to a Parquet file.
// Uses streaming writes to avoid accumulating all rows in memory.
type parquetArchiveWriter struct {
	writer *pq.GenericWriter[ParquetMetricRow]
	bw     *bufio.Writer
	f      *os.File
	batch  []ParquetMetricRow // reusable batch buffer
	count  int
}

const parquetBatchSize = 1024

// newParquetArchiveWriter creates a streaming Parquet writer with Zstd compression.
func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
		return nil, fmt.Errorf("creating archive directory: %w", err)
	}

	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return nil, fmt.Errorf("creating parquet file: %w", err)
	}

	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer

	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
		pq.Compression(&pq.Zstd),
	)

	return &parquetArchiveWriter{
		writer: writer,
		bw:     bw,
		f:      f,
		batch:  make([]ParquetMetricRow, 0, parquetBatchSize),
	}, nil
}

// WriteCheckpointFile streams a CheckpointFile tree directly to Parquet rows,
// writing metrics in sorted order without materializing all rows in memory.
// Produces one row group per call (typically one host's data).
func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {
	w.writeLevel(cf, cluster, hostname, scope, scopeID)

	// Flush remaining batch
	if len(w.batch) > 0 {
		if _, err := w.writer.Write(w.batch); err != nil {
			return fmt.Errorf("writing parquet rows: %w", err)
		}
		w.count += len(w.batch)
		w.batch = w.batch[:0]
	}

	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}

	return nil
}

// writeLevel recursively writes metrics from a CheckpointFile level.
// Metric names and child names are sorted for deterministic, compression-friendly output.
func (w *parquetArchiveWriter) writeLevel(cf *CheckpointFile, cluster, hostname, scope, scopeID string) {
	// Sort metric names for deterministic order
	metricNames := make([]string, 0, len(cf.Metrics))
	for name := range cf.Metrics {
		metricNames = append(metricNames, name)
	}
	sort.Strings(metricNames)

	for _, metricName := range metricNames {
		cm := cf.Metrics[metricName]
		ts := cm.Start
		for _, v := range cm.Data {
			if !v.IsNaN() {
				w.batch = append(w.batch, ParquetMetricRow{
					Cluster:   cluster,
					Hostname:  hostname,
					Metric:    metricName,
					Scope:     scope,
					ScopeID:   scopeID,
					Timestamp: ts,
					Frequency: cm.Frequency,
					Value:     float32(v),
				})

				if len(w.batch) >= parquetBatchSize {
					w.writer.Write(w.batch)
					w.count += len(w.batch)
					w.batch = w.batch[:0]
				}
			}
			ts += cm.Frequency
		}
	}

	// Sort child names for deterministic order
	childNames := make([]string, 0, len(cf.Children))
	for name := range cf.Children {
		childNames = append(childNames, name)
	}
	sort.Strings(childNames)

	for _, childName := range childNames {
		childScope, childScopeID := parseScopeFromName(childName)
		w.writeLevel(cf.Children[childName], cluster, hostname, childScope, childScopeID)
	}
}

// Close finalises the Parquet file (footer, buffered I/O, file handle).
func (w *parquetArchiveWriter) Close() error {
	if err := w.writer.Close(); err != nil {
		w.f.Close()
		return fmt.Errorf("closing parquet writer: %w", err)
	}

	if err := w.bw.Flush(); err != nil {
		w.f.Close()
		return fmt.Errorf("flushing parquet file: %w", err)
	}

	return w.f.Close()
}

// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
// before converting it to Parquet format.
func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ext := filepath.Ext(filename)
	switch ext {
	case ".json":
		cf := &CheckpointFile{}
		br := bufio.NewReader(f)
		if err := json.NewDecoder(br).Decode(cf); err != nil {
			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
		}
		return cf, nil

	case ".bin":
		br := bufio.NewReader(f)
		var magic uint32
		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
		}
		if magic != snapFileMagic {
			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
		}
		var fileFrom, fileTo int64
		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
		}
		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
		}
		cf, err := readBinaryLevel(br)
		if err != nil {
			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
		}
		cf.From = fileFrom
		cf.To = fileTo
		return cf, nil

	default:
		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
	}
}

// loadCheckpointFiles reads checkpoint files for a host directory and returns
// the loaded CheckpointFiles and their filenames. Processes one file at a time
// to avoid holding all checkpoint data in memory simultaneously.
func loadCheckpointFiles(dir string, from int64) ([]*CheckpointFile, []string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, nil, err
	}

	files, err := findFiles(entries, from, false)
	if err != nil {
		return nil, nil, err
	}

	if len(files) == 0 {
		return nil, nil, nil
	}

	var checkpoints []*CheckpointFile
	var processedFiles []string

	for _, checkpoint := range files {
		filename := filepath.Join(dir, checkpoint)
		cf, err := loadCheckpointFileFromDisk(filename)
		if err != nil {
			continue
		}
		checkpoints = append(checkpoints, cf)
		processedFiles = append(processedFiles, checkpoint)
	}

	return checkpoints, processedFiles, nil
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/parquetArchive.go has been updated successfully.

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"bufio"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	pq "github.com/parquet-go/parquet-go"
)

// ParquetMetricRow is the long-format schema for archived metric data.
// One row per (host, metric, scope, scope_id, timestamp) data point.
// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
type ParquetMetricRow struct {
	Cluster   string  `parquet:"cluster"`
	Hostname  string  `parquet:"hostname"`
	Metric    string  `parquet:"metric"`
	Scope     string  `parquet:"scope"`
	ScopeID   string  `parquet:"scope_id"`
	Timestamp int64   `parquet:"timestamp"`
	Frequency int64   `parquet:"frequency"`
	Value     float32 `parquet:"value"`
}

// flattenCheckpointFile recursively converts a CheckpointFile tree into Parquet rows.
// The scope path is built from the hierarchy: host level is "node", then child names
// map to scope/scope_id (e.g., "socket0" → scope="socket", scope_id="0").
func flattenCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string, rows []ParquetMetricRow) []ParquetMetricRow {
	for metricName, cm := range cf.Metrics {
		ts := cm.Start
		for _, v := range cm.Data {
			if !v.IsNaN() {
				rows = append(rows, ParquetMetricRow{
					Cluster:   cluster,
					Hostname:  hostname,
					Metric:    metricName,
					Scope:     scope,
					ScopeID:   scopeID,
					Timestamp: ts,
					Frequency: cm.Frequency,
					Value:     float32(v),
				})
			}
			ts += cm.Frequency
		}
	}

	for childName, childCf := range cf.Children {
		childScope, childScopeID := parseScopeFromName(childName)
		rows = flattenCheckpointFile(childCf, cluster, hostname, childScope, childScopeID, rows)
	}

	return rows
}

// parseScopeFromName infers scope and scope_id from a child level name.
// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
// "a0" (accelerator) → ("accelerator", "0").
// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
func parseScopeFromName(name string) (string, string) {
	prefixes := []struct {
		prefix string
		scope  string
	}{
		{"socket", "socket"},
		{"memoryDomain", "memoryDomain"},
		{"core", "core"},
		{"hwthread", "hwthread"},
		{"cpu", "hwthread"},
		{"accelerator", "accelerator"},
	}

	for _, p := range prefixes {
		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
			id := name[len(p.prefix):]
			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
				return p.scope, id
			}
		}
	}

	return name, ""
}

// parquetArchiveWriter supports incremental writes to a Parquet file.
// Each call to WriteHostRows writes one row group (typically one host's data),
// avoiding accumulation of all rows in memory.
type parquetArchiveWriter struct {
	writer *pq.GenericWriter[ParquetMetricRow]
	bw     *bufio.Writer
	f      *os.File
	count  int
}

// newParquetArchiveWriter creates a streaming Parquet writer with Zstd compression.
func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
		return nil, fmt.Errorf("creating archive directory: %w", err)
	}

	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return nil, fmt.Errorf("creating parquet file: %w", err)
	}

	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer

	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
		pq.Compression(&pq.Zstd),
	)

	return &parquetArchiveWriter{writer: writer, bw: bw, f: f}, nil
}

// WriteHostRows sorts rows by (metric, timestamp) in-place, writes them,
// and flushes to create a separate row group.
func (w *parquetArchiveWriter) WriteHostRows(rows []ParquetMetricRow) error {
	sort.Slice(rows, func(i, j int) bool {
		if rows[i].Metric != rows[j].Metric {
			return rows[i].Metric < rows[j].Metric
		}
		return rows[i].Timestamp < rows[j].Timestamp
	})

	if _, err := w.writer.Write(rows); err != nil {
		return fmt.Errorf("writing parquet rows: %w", err)
	}

	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}

	w.count += len(rows)
	return nil
}

// Close finalises the Parquet file (footer, buffered I/O, file handle).
func (w *parquetArchiveWriter) Close() error {
	if err := w.writer.Close(); err != nil {
		w.f.Close()
		return fmt.Errorf("closing parquet writer: %w", err)
	}

	if err := w.bw.Flush(); err != nil {
		w.f.Close()
		return fmt.Errorf("flushing parquet file: %w", err)
	}

	return w.f.Close()
}

// sortParquetRows sorts rows by (metric, timestamp) in-place.
func sortParquetRows(rows []ParquetMetricRow) {
	sort.Slice(rows, func(i, j int) bool {
		if rows[i].Metric != rows[j].Metric {
			return rows[i].Metric < rows[j].Metric
		}
		return rows[i].Timestamp < rows[j].Timestamp
	})
}

// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
// before converting it to Parquet format.
func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ext := filepath.Ext(filename)
	switch ext {
	case ".json":
		cf := &CheckpointFile{}
		br := bufio.NewReader(f)
		if err := json.NewDecoder(br).Decode(cf); err != nil {
			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
		}
		return cf, nil

	case ".bin":
		br := bufio.NewReader(f)
		var magic uint32
		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
		}
		if magic != snapFileMagic {
			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
		}
		var fileFrom, fileTo int64
		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
		}
		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
		}
		cf, err := readBinaryLevel(br)
		if err != nil {
			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
		}
		cf.From = fileFrom
		cf.To = fileTo
		return cf, nil

	default:
		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
	}
}

// estimateRowCount estimates the number of Parquet rows a CheckpointFile will produce.
// Used for pre-allocating the rows slice to avoid repeated append doubling.
func estimateRowCount(cf *CheckpointFile) int {
	n := 0
	for _, cm := range cf.Metrics {
		n += len(cm.Data)
	}
	for _, child := range cf.Children {
		n += estimateRowCount(child)
	}
	return n
}

// archiveCheckpointsToParquet reads checkpoint files for a host directory,
// converts them to Parquet rows. Returns the rows and filenames that were processed.
func archiveCheckpointsToParquet(dir, cluster, host string, from int64) ([]ParquetMetricRow, []string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, nil, err
	}

	files, err := findFiles(entries, from, false)
	if err != nil {
		return nil, nil, err
	}

	if len(files) == 0 {
		return nil, nil, nil
	}

	// First pass: load checkpoints and estimate total rows for pre-allocation.
	type loaded struct {
		cf       *CheckpointFile
		filename string
	}
	var checkpoints []loaded
	totalEstimate := 0

	for _, checkpoint := range files {
		filename := filepath.Join(dir, checkpoint)
		cf, err := loadCheckpointFileFromDisk(filename)
		if err != nil {
			cclog.Warnf("[METRICSTORE]> skipping unreadable checkpoint %s: %v", filename, err)
			continue
		}
		totalEstimate += estimateRowCount(cf)
		checkpoints = append(checkpoints, loaded{cf: cf, filename: checkpoint})
	}

	if len(checkpoints) == 0 {
		return nil, nil, nil
	}

	rows := make([]ParquetMetricRow, 0, totalEstimate)
	processedFiles := make([]string, 0, len(checkpoints))

	for _, cp := range checkpoints {
		rows = flattenCheckpointFile(cp.cf, cluster, host, "node", "", rows)
		processedFiles = append(processedFiles, cp.filename)
	}

	return rows, processedFiles, nil
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"bufio"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"

	pq "github.com/parquet-go/parquet-go"
)

// ParquetMetricRow is the long-format schema for archived metric data.
// One row per (host, metric, scope, scope_id, timestamp) data point.
// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
type ParquetMetricRow struct {
	Cluster   string  `parquet:"cluster"`
	Hostname  string  `parquet:"hostname"`
	Metric    string  `parquet:"metric"`
	Scope     string  `parquet:"scope"`
	ScopeID   string  `parquet:"scope_id"`
	Timestamp int64   `parquet:"timestamp"`
	Frequency int64   `parquet:"frequency"`
	Value     float32 `parquet:"value"`
}

// parseScopeFromName infers scope and scope_id from a child level name.
// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
// "a0" (accelerator) → ("accelerator", "0").
// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
func parseScopeFromName(name string) (string, string) {
	prefixes := []struct {
		prefix string
		scope  string
	}{
		{"socket", "socket"},
		{"memoryDomain", "memoryDomain"},
		{"core", "core"},
		{"hwthread", "hwthread"},
		{"cpu", "hwthread"},
		{"accelerator", "accelerator"},
	}

	for _, p := range prefixes {
		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
			id := name[len(p.prefix):]
			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
				return p.scope, id
			}
		}
	}

	return name, ""
}

// parquetArchiveWriter supports incremental writes to a Parquet file.
// Uses streaming writes to avoid accumulating all rows in memory.
type parquetArchiveWriter struct {
	writer *pq.GenericWriter[ParquetMetricRow]
	bw     *bufio.Writer
	f      *os.File
	batch  []ParquetMetricRow // reusable batch buffer
	count  int
}

const parquetBatchSize = 1024

// newParquetArchiveWriter creates a streaming Parquet writer with Zstd compression.
func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
		return nil, fmt.Errorf("creating archive directory: %w", err)
	}

	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return nil, fmt.Errorf("creating parquet file: %w", err)
	}

	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer

	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
		pq.Compression(&pq.Zstd),
	)

	return &parquetArchiveWriter{
		writer: writer,
		bw:     bw,
		f:      f,
		batch:  make([]ParquetMetricRow, 0, parquetBatchSize),
	}, nil
}

// WriteCheckpointFile streams a CheckpointFile tree directly to Parquet rows,
// writing metrics in sorted order without materializing all rows in memory.
// Produces one row group per call (typically one host's data).
func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {
	w.writeLevel(cf, cluster, hostname, scope, scopeID)

	// Flush remaining batch
	if len(w.batch) > 0 {
		if _, err := w.writer.Write(w.batch); err != nil {
			return fmt.Errorf("writing parquet rows: %w", err)
		}
		w.count += len(w.batch)
		w.batch = w.batch[:0]
	}

	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}

	return nil
}

// writeLevel recursively writes metrics from a CheckpointFile level.
// Metric names and child names are sorted for deterministic, compression-friendly output.
func (w *parquetArchiveWriter) writeLevel(cf *CheckpointFile, cluster, hostname, scope, scopeID string) {
	// Sort metric names for deterministic order
	metricNames := make([]string, 0, len(cf.Metrics))
	for name := range cf.Metrics {
		metricNames = append(metricNames, name)
	}
	sort.Strings(metricNames)

	for _, metricName := range metricNames {
		cm := cf.Metrics[metricName]
		ts := cm.Start
		for _, v := range cm.Data {
			if !v.IsNaN() {
				w.batch = append(w.batch, ParquetMetricRow{
					Cluster:   cluster,
					Hostname:  hostname,
					Metric:    metricName,
					Scope:     scope,
					ScopeID:   scopeID,
					Timestamp: ts,
					Frequency: cm.Frequency,
					Value:     float32(v),
				})

				if len(w.batch) >= parquetBatchSize {
					w.writer.Write(w.batch)
					w.count += len(w.batch)
					w.batch = w.batch[:0]
				}
			}
			ts += cm.Frequency
		}
	}

	// Sort child names for deterministic order
	childNames := make([]string, 0, len(cf.Children))
	for name := range cf.Children {
		childNames = append(childNames, name)
	}
	sort.Strings(childNames)

	for _, childName := range childNames {
		childScope, childScopeID := parseScopeFromName(childName)
		w.writeLevel(cf.Children[childName], cluster, hostname, childScope, childScopeID)
	}
}

// Close finalises the Parquet file (footer, buffered I/O, file handle).
func (w *parquetArchiveWriter) Close() error {
	if err := w.writer.Close(); err != nil {
		w.f.Close()
		return fmt.Errorf("closing parquet writer: %w", err)
	}

	if err := w.bw.Flush(); err != nil {
		w.f.Close()
		return fmt.Errorf("flushing parquet file: %w", err)
	}

	return w.f.Close()
}

// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
// before converting it to Parquet format.
func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ext := filepath.Ext(filename)
	switch ext {
	case ".json":
		cf := &CheckpointFile{}
		br := bufio.NewReader(f)
		if err := json.NewDecoder(br).Decode(cf); err != nil {
			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
		}
		return cf, nil

	case ".bin":
		br := bufio.NewReader(f)
		var magic uint32
		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
		}
		if magic != snapFileMagic {
			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
		}
		var fileFrom, fileTo int64
		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
		}
		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
		}
		cf, err := readBinaryLevel(br)
		if err != nil {
			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
		}
		cf.From = fileFrom
		cf.To = fileTo
		return cf, nil

	default:
		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
	}
}

// loadCheckpointFiles reads checkpoint files for a host directory and returns
// the loaded CheckpointFiles and their filenames. Processes one file at a time
// to avoid holding all checkpoint data in memory simultaneously.
func loadCheckpointFiles(dir string, from int64) ([]*CheckpointFile, []string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, nil, err
	}

	files, err := findFiles(entries, from, false)
	if err != nil {
		return nil, nil, err
	}

	if len(files) == 0 {
		return nil, nil, nil
	}

	var checkpoints []*CheckpointFile
	var processedFiles []string

	for _, checkpoint := range files {
		filename := filepath.Join(dir, checkpoint)
		cf, err := loadCheckpointFileFromDisk(filename)
		if err != nil {
			continue
		}
		checkpoints = append(checkpoints, cf)
		processedFiles = append(processedFiles, checkpoint)
	}

	return checkpoints, processedFiles, nil
}

-  cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"

-// flattenCheckpointFile recursively converts a CheckpointFile tree into Parquet rows.

-// The scope path is built from the hierarchy: host level is "node", then child names

-// map to scope/scope_id (e.g., "socket0" → scope="socket", scope_id="0").

-func flattenCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string, rows []ParquetMetricRow) []ParquetMetricRow {

-  for metricName, cm := range cf.Metrics {

-        rows = append(rows, ParquetMetricRow{

-  for childName, childCf := range cf.Children {

-    childScope, childScopeID := parseScopeFromName(childName)

-    rows = flattenCheckpointFile(childCf, cluster, hostname, childScope, childScopeID, rows)

// "a0" (accelerator) → ("accelerator", "0").

-// Each call to WriteHostRows writes one row group (typically one host's data),

-// avoiding accumulation of all rows in memory.

+// Uses streaming writes to avoid accumulating all rows in memory.

+  batch  []ParquetMetricRow // reusable batch buffer

-  return &parquetArchiveWriter{writer: writer, bw: bw, f: f}, nil

+    batch:  make([]ParquetMetricRow, 0, parquetBatchSize),

-// WriteHostRows sorts rows by (metric, timestamp) in-place, writes them,

-// and flushes to create a separate row group.

-func (w *parquetArchiveWriter) WriteHostRows(rows []ParquetMetricRow) error {

-  sort.Slice(rows, func(i, j int) bool {

-    if rows[i].Metric != rows[j].Metric {

-      return rows[i].Metric < rows[j].Metric

-    return rows[i].Timestamp < rows[j].Timestamp

+// WriteCheckpointFile streams a CheckpointFile tree directly to Parquet rows,

+// writing metrics in sorted order without materializing all rows in memory.

+// Produces one row group per call (typically one host's data).

+func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {

+  w.writeLevel(cf, cluster, hostname, scope, scopeID)

-  if _, err := w.writer.Write(rows); err != nil {

-    return fmt.Errorf("writing parquet rows: %w", err)

+    if _, err := w.writer.Write(w.batch); err != nil {

+      return fmt.Errorf("writing parquet rows: %w", err)

if err := w.writer.Flush(); err != nil {

+// writeLevel recursively writes metrics from a CheckpointFile level.

+// Metric names and child names are sorted for deterministic, compression-friendly output.

+func (w *parquetArchiveWriter) writeLevel(cf *CheckpointFile, cluster, hostname, scope, scopeID string) {

+  // Sort metric names for deterministic order

+  metricNames := make([]string, 0, len(cf.Metrics))

+    metricNames = append(metricNames, name)

+  for _, metricName := range metricNames {

+        w.batch = append(w.batch, ParquetMetricRow{

+        if len(w.batch) >= parquetBatchSize {

+  // Sort child names for deterministic order

+  childNames := make([]string, 0, len(cf.Children))

+    childNames = append(childNames, name)

+  for _, childName := range childNames {

+    childScope, childScopeID := parseScopeFromName(childName)

+    w.writeLevel(cf.Children[childName], cluster, hostname, childScope, childScopeID)

func (w *parquetArchiveWriter) Close() error {

if err := w.writer.Close(); err != nil {

-// sortParquetRows sorts rows by (metric, timestamp) in-place.

-func sortParquetRows(rows []ParquetMetricRow) {

-  sort.Slice(rows, func(i, j int) bool {

-    if rows[i].Metric != rows[j].Metric {

-      return rows[i].Metric < rows[j].Metric

-    return rows[i].Timestamp < rows[j].Timestamp

// before converting it to Parquet format.

-// estimateRowCount estimates the number of Parquet rows a CheckpointFile will produce.

-// Used for pre-allocating the rows slice to avoid repeated append doubling.

-func estimateRowCount(cf *CheckpointFile) int {

-// archiveCheckpointsToParquet reads checkpoint files for a host directory,

-// converts them to Parquet rows. Returns the rows and filenames that were processed.

-func archiveCheckpointsToParquet(dir, cluster, host string, from int64) ([]ParquetMetricRow, []string, error) {

+// loadCheckpointFiles reads checkpoint files for a host directory and returns

+// the loaded CheckpointFiles and their filenames. Processes one file at a time

+// to avoid holding all checkpoint data in memory simultaneously.

+func loadCheckpointFiles(dir string, from int64) ([]*CheckpointFile, []string, error) {

-  // First pass: load checkpoints and estimate total rows for pre-allocation.

filename := filepath.Join(dir, checkpoint)

cf, err := loadCheckpointFileFromDisk(filename)

-      cclog.Warnf("[METRICSTORE]> skipping unreadable checkpoint %s: %v", filename, err)

-    totalEstimate += estimateRowCount(cf)

-    checkpoints = append(checkpoints, loaded{cf: cf, filename: checkpoint})

+    checkpoints = append(checkpoints, cf)

+    processedFiles = append(processedFiles, checkpoint)

-  rows := make([]ParquetMetricRow, 0, totalEstimate)

-  processedFiles := make([]string, 0, len(checkpoints))

-    rows = flattenCheckpointFile(cp.cf, cluster, host, "node", "", rows)

-    processedFiles = append(processedFiles, cp.filename)

+  return checkpoints, processedFiles, nil


Now update `archive.go` to use the new streaming approach:


// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Each host's rows are written as a separate row group to avoid accumulating
// all data in memory at once.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Stream per-host rows to parquet writer via worker pool
		type hostResult struct {
			rows  []ParquetMetricRow
			files []string // checkpoint filenames to delete after successful write
			dir   string   // checkpoint directory for this host
		}

		results := make(chan hostResult, len(hostEntries))
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					rows, files, err := archiveCheckpointsToParquet(item.dir, cluster, item.host, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(rows) > 0 {
						results <- hostResult{rows: rows, files: files, dir: item.dir}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's rows as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				sortParquetRows(r.rows)
				if err := writer.WriteHostRows(r.rows); err != nil {
					writeErr = err
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go has been updated successfully.

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {

		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir            string
		cluster, host  string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Each host's rows are written as a separate row group to avoid accumulating
// all data in memory at once.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			return totalFiles, err
		}

		// Stream per-host rows to parquet writer via worker pool
		type hostResult struct {
			rows  []ParquetMetricRow
			files []string // checkpoint filenames to delete after successful write
			dir   string   // checkpoint directory for this host
		}

		results := make(chan hostResult, len(hostEntries))
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					rows, files, err := archiveCheckpointsToParquet(item.dir, cluster, item.host, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(rows) > 0 {
						results <- hostResult{rows: rows, files: files, dir: item.dir}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's rows as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				sortParquetRows(r.rows)
				if err := writer.WriteHostRows(r.rows); err != nil {
					writeErr = err
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
	return totalFiles, nil
}

-// Each host's rows are written as a separate row group to avoid accumulating

+// Workers load checkpoint files from disk and send CheckpointFile trees on a

+// back-pressured channel. The main thread streams each tree directly to Parquet

+// rows without materializing all rows in memory.

-    // Stream per-host rows to parquet writer via worker pool

+    // Workers load checkpoint files from disk; main thread writes to parquet.

-      files []string // checkpoint filenames to delete after successful write

-      dir   string   // checkpoint directory for this host

+      files       []string // checkpoint filenames to delete after successful write

+      dir         string   // checkpoint directory for this host

-    results := make(chan hostResult, len(hostEntries))

+    // Small buffer provides back-pressure: at most NumWorkers+2 results in flight.

+    results := make(chan hostResult, 2)

-          rows, files, err := archiveCheckpointsToParquet(item.dir, cluster, item.host, from)

+          checkpoints, files, err := loadCheckpointFiles(item.dir, from)

-            results <- hostResult{rows: rows, files: files, dir: item.dir}

+              checkpoints: checkpoints,

-    // Open streaming writer and write each host's rows as a row group

+    // Open streaming writer and write each host's checkpoint files as a row group

-        if err := writer.WriteHostRows(r.rows); err != nil {

+        // Stream each checkpoint file directly to parquet rows.

+        // Each checkpoint is processed and discarded before the next.

+        for _, cf := range r.checkpoints {

+          if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {


The diagnostics show `archive.go` still has old code — that's from the diff showing my old_string matching. Let me check and also fix the test file.


pkg/metricstore/parquetArchive_test.go:72:	rows := flattenCheckpointFile(cf, "fritz", "node001", "node", "", nil)
pkg/metricstore/parquetArchive_test.go:156:	rows, files, err := archiveCheckpointsToParquet(cpDir, "testcluster", "node001", 2000)
pkg/metricstore/parquetArchive_test.go:169:	sortParquetRows(rows)
pkg/metricstore/parquetArchive_test.go:170:	if err := writer.WriteHostRows(rows); err != nil {


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"encoding/json"
    10→	"os"
    11→	"path/filepath"
    12→	"testing"
    13→
    14→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    15→	pq "github.com/parquet-go/parquet-go"
    16→)
    17→
    18→func TestParseScopeFromName(t *testing.T) {
    19→	tests := []struct {
    20→		name      string
    21→		wantScope string
    22→		wantID    string
    23→	}{
    24→		{"socket0", "socket", "0"},
    25→		{"socket12", "socket", "12"},
    26→		{"core0", "core", "0"},
    27→		{"core127", "core", "127"},
    28→		{"cpu0", "hwthread", "0"},
    29→		{"hwthread5", "hwthread", "5"},
    30→		{"memoryDomain0", "memoryDomain", "0"},
    31→		{"accelerator0", "accelerator", "0"},
    32→		{"unknown", "unknown", ""},
    33→		{"socketX", "socketX", ""}, // not numeric suffix
    34→	}
    35→
    36→	for _, tt := range tests {
    37→		t.Run(tt.name, func(t *testing.T) {
    38→			scope, id := parseScopeFromName(tt.name)
    39→			if scope != tt.wantScope || id != tt.wantID {
    40→				t.Errorf("parseScopeFromName(%q) = (%q, %q), want (%q, %q)",
    41→					tt.name, scope, id, tt.wantScope, tt.wantID)
    42→			}
    43→		})
    44→	}
    45→}
    46→
    47→func TestFlattenCheckpointFile(t *testing.T) {
    48→	cf := &CheckpointFile{
    49→		From: 1000,
    50→		To:   1060,
    51→		Metrics: map[string]*CheckpointMetrics{
    52→			"cpu_load": {
    53→				Frequency: 60,
    54→				Start:     1000,
    55→				Data:      []schema.Float{0.5, 0.7, schema.NaN},
    56→			},
    57→		},
    58→		Children: map[string]*CheckpointFile{
    59→			"socket0": {
    60→				Metrics: map[string]*CheckpointMetrics{
    61→					"mem_bw": {
    62→						Frequency: 60,
    63→						Start:     1000,
    64→						Data:      []schema.Float{100.0, schema.NaN, 200.0},
    65→					},
    66→				},
    67→				Children: make(map[string]*CheckpointFile),
    68→			},
    69→		},
    70→	}
    71→
    72→	rows := flattenCheckpointFile(cf, "fritz", "node001", "node", "", nil)
    73→
    74→	// cpu_load: 2 non-NaN values at node scope
    75→	// mem_bw: 2 non-NaN values at socket0 scope
    76→	if len(rows) != 4 {
    77→		t.Fatalf("expected 4 rows, got %d", len(rows))
    78→	}
    79→
    80→	// Verify a node-scope row
    81→	found := false
    82→	for _, r := range rows {
    83→		if r.Metric == "cpu_load" && r.Timestamp == 1000 {
    84→			found = true
    85→			if r.Cluster != "fritz" || r.Hostname != "node001" || r.Scope != "node" || r.Value != 0.5 {
    86→				t.Errorf("unexpected row: %+v", r)
    87→			}
    88→		}
    89→	}
    90→	if !found {
    91→		t.Error("expected cpu_load row at timestamp 1000")
    92→	}
    93→
    94→	// Verify a socket-scope row
    95→	found = false
    96→	for _, r := range rows {
    97→		if r.Metric == "mem_bw" && r.Scope == "socket" && r.ScopeID == "0" {
    98→			found = true
    99→		}
   100→	}
   101→	if !found {
   102→		t.Error("expected mem_bw row with scope=socket, scope_id=0")
   103→	}
   104→}
   105→
   106→func TestParquetArchiveRoundtrip(t *testing.T) {
   107→	tmpDir := t.TempDir()
   108→
   109→	// Create checkpoint files on disk (JSON format)
   110→	cpDir := filepath.Join(tmpDir, "checkpoints", "testcluster", "node001")
   111→	if err := os.MkdirAll(cpDir, 0o755); err != nil {
   112→		t.Fatal(err)
   113→	}
   114→
   115→	cf := &CheckpointFile{
   116→		From: 1000,
   117→		To:   1180,
   118→		Metrics: map[string]*CheckpointMetrics{
   119→			"cpu_load": {
   120→				Frequency: 60,
   121→				Start:     1000,
   122→				Data:      []schema.Float{0.5, 0.7, 0.9},
   123→			},
   124→			"mem_used": {
   125→				Frequency: 60,
   126→				Start:     1000,
   127→				Data:      []schema.Float{45.0, 46.0, 47.0},
   128→			},
   129→		},
   130→		Children: map[string]*CheckpointFile{
   131→			"socket0": {
   132→				Metrics: map[string]*CheckpointMetrics{
   133→					"mem_bw": {
   134→						Frequency: 60,
   135→						Start:     1000,
   136→						Data:      []schema.Float{100.0, 110.0, 120.0},
   137→					},
   138→				},
   139→				Children: make(map[string]*CheckpointFile),
   140→			},
   141→		},
   142→	}
   143→
   144→	// Write JSON checkpoint
   145→	cpFile := filepath.Join(cpDir, "1000.json")
   146→	data, err := json.Marshal(cf)
   147→	if err != nil {
   148→		t.Fatal(err)
   149→	}
   150→	if err := os.WriteFile(cpFile, data, 0o644); err != nil {
   151→		t.Fatal(err)
   152→	}
   153→
   154→	// Archive to Parquet
   155→	archiveDir := filepath.Join(tmpDir, "archive")
   156→	rows, files, err := archiveCheckpointsToParquet(cpDir, "testcluster", "node001", 2000)
   157→	if err != nil {
   158→		t.Fatal(err)
   159→	}
   160→	if len(files) != 1 || files[0] != "1000.json" {
   161→		t.Fatalf("expected 1 file, got %v", files)
   162→	}
   163→
   164→	parquetFile := filepath.Join(archiveDir, "testcluster", "1000.parquet")
   165→	writer, err := newParquetArchiveWriter(parquetFile)
   166→	if err != nil {
   167→		t.Fatal(err)
   168→	}
   169→	sortParquetRows(rows)
   170→	if err := writer.WriteHostRows(rows); err != nil {
   171→		t.Fatal(err)
   172→	}
   173→	if err := writer.Close(); err != nil {
   174→		t.Fatal(err)
   175→	}
   176→
   177→	// Read back and verify
   178→	f, err := os.Open(parquetFile)
   179→	if err != nil {
   180→		t.Fatal(err)
   181→	}
   182→	defer f.Close()
   183→
   184→	stat, _ := f.Stat()
   185→	pf, err := pq.OpenFile(f, stat.Size())
   186→	if err != nil {
   187→		t.Fatal(err)
   188→	}
   189→
   190→	reader := pq.NewGenericReader[ParquetMetricRow](pf)
   191→	readRows := make([]ParquetMetricRow, 100)
   192→	n, err := reader.Read(readRows)
   193→	if err != nil && n == 0 {
   194→		t.Fatal(err)
   195→	}
   196→	readRows = readRows[:n]
   197→	reader.Close()
   198→
   199→	// We expect: cpu_load(3) + mem_used(3) + mem_bw(3) = 9 rows
   200→	if n != 9 {
   201→		t.Fatalf("expected 9 rows in parquet file, got %d", n)
   202→	}
   203→
   204→	// Verify cluster and hostname are set correctly
   205→	for _, r := range readRows {
   206→		if r.Cluster != "testcluster" {
   207→			t.Errorf("expected cluster=testcluster, got %s", r.Cluster)
   208→		}
   209→		if r.Hostname != "node001" {
   210→			t.Errorf("expected hostname=node001, got %s", r.Hostname)
   211→		}
   212→	}
   213→
   214→	// Verify parquet file is smaller than JSON (compression working)
   215→	if stat.Size() == 0 {
   216→		t.Error("parquet file is empty")
   217→	}
   218→
   219→	t.Logf("Parquet file size: %d bytes for %d rows", stat.Size(), n)
   220→}
   221→
   222→func TestLoadCheckpointFileFromDisk_JSON(t *testing.T) {
   223→	tmpDir := t.TempDir()
   224→
   225→	cf := &CheckpointFile{
   226→		From: 1000,
   227→		To:   1060,
   228→		Metrics: map[string]*CheckpointMetrics{
   229→			"test_metric": {
   230→				Frequency: 60,
   231→				Start:     1000,
   232→				Data:      []schema.Float{1.0, 2.0, 3.0},
   233→			},
   234→		},
   235→		Children: make(map[string]*CheckpointFile),
   236→	}
   237→
   238→	filename := filepath.Join(tmpDir, "1000.json")
   239→	data, err := json.Marshal(cf)
   240→	if err != nil {
   241→		t.Fatal(err)
   242→	}
   243→	if err := os.WriteFile(filename, data, 0o644); err != nil {
   244→		t.Fatal(err)
   245→	}
   246→
   247→	loaded, err := loadCheckpointFileFromDisk(filename)
   248→	if err != nil {
   249→		t.Fatal(err)
   250→	}
   251→
   252→	if loaded.From != 1000 || loaded.To != 1060 {
   253→		t.Errorf("expected From=1000, To=1060, got From=%d, To=%d", loaded.From, loaded.To)
   254→	}
   255→
   256→	m, ok := loaded.Metrics["test_metric"]
   257→	if !ok {
   258→		t.Fatal("expected test_metric in loaded checkpoint")
   259→	}
   260→	if m.Frequency != 60 || m.Start != 1000 || len(m.Data) != 3 {
   261→		t.Errorf("unexpected metric data: %+v", m)
   262→	}
   263→}
   264→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"encoding/json"
	"os"
	"path/filepath"
	"testing"

	"github.com/ClusterCockpit/cc-lib/v2/schema"
	pq "github.com/parquet-go/parquet-go"
)

func TestParseScopeFromName(t *testing.T) {
	tests := []struct {
		name      string
		wantScope string
		wantID    string
	}{
		{"socket0", "socket", "0"},
		{"socket12", "socket", "12"},
		{"core0", "core", "0"},
		{"core127", "core", "127"},
		{"cpu0", "hwthread", "0"},
		{"hwthread5", "hwthread", "5"},
		{"memoryDomain0", "memoryDomain", "0"},
		{"accelerator0", "accelerator", "0"},
		{"unknown", "unknown", ""},
		{"socketX", "socketX", ""}, // not numeric suffix
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			scope, id := parseScopeFromName(tt.name)
			if scope != tt.wantScope || id != tt.wantID {
				t.Errorf("parseScopeFromName(%q) = (%q, %q), want (%q, %q)",
					tt.name, scope, id, tt.wantScope, tt.wantID)
			}
		})
	}
}

func TestFlattenCheckpointFile(t *testing.T) {
	cf := &CheckpointFile{
		From: 1000,
		To:   1060,
		Metrics: map[string]*CheckpointMetrics{
			"cpu_load": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{0.5, 0.7, schema.NaN},
			},
		},
		Children: map[string]*CheckpointFile{
			"socket0": {
				Metrics: map[string]*CheckpointMetrics{
					"mem_bw": {
						Frequency: 60,
						Start:     1000,
						Data:      []schema.Float{100.0, schema.NaN, 200.0},
					},
				},
				Children: make(map[string]*CheckpointFile),
			},
		},
	}

	rows := flattenCheckpointFile(cf, "fritz", "node001", "node", "", nil)

	// cpu_load: 2 non-NaN values at node scope
	// mem_bw: 2 non-NaN values at socket0 scope
	if len(rows) != 4 {
		t.Fatalf("expected 4 rows, got %d", len(rows))
	}

	// Verify a node-scope row
	found := false
	for _, r := range rows {
		if r.Metric == "cpu_load" && r.Timestamp == 1000 {
			found = true
			if r.Cluster != "fritz" || r.Hostname != "node001" || r.Scope != "node" || r.Value != 0.5 {
				t.Errorf("unexpected row: %+v", r)
			}
		}
	}
	if !found {
		t.Error("expected cpu_load row at timestamp 1000")
	}

	// Verify a socket-scope row
	found = false
	for _, r := range rows {
		if r.Metric == "mem_bw" && r.Scope == "socket" && r.ScopeID == "0" {
			found = true
		}
	}
	if !found {
		t.Error("expected mem_bw row with scope=socket, scope_id=0")
	}
}

func TestParquetArchiveRoundtrip(t *testing.T) {
	tmpDir := t.TempDir()

	// Create checkpoint files on disk (JSON format)
	cpDir := filepath.Join(tmpDir, "checkpoints", "testcluster", "node001")
	if err := os.MkdirAll(cpDir, 0o755); err != nil {
		t.Fatal(err)
	}

	cf := &CheckpointFile{
		From: 1000,
		To:   1180,
		Metrics: map[string]*CheckpointMetrics{
			"cpu_load": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{0.5, 0.7, 0.9},
			},
			"mem_used": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{45.0, 46.0, 47.0},
			},
		},
		Children: map[string]*CheckpointFile{
			"socket0": {
				Metrics: map[string]*CheckpointMetrics{
					"mem_bw": {
						Frequency: 60,
						Start:     1000,
						Data:      []schema.Float{100.0, 110.0, 120.0},
					},
				},
				Children: make(map[string]*CheckpointFile),
			},
		},
	}

	// Write JSON checkpoint
	cpFile := filepath.Join(cpDir, "1000.json")
	data, err := json.Marshal(cf)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(cpFile, data, 0o644); err != nil {
		t.Fatal(err)
	}

	// Archive to Parquet
	archiveDir := filepath.Join(tmpDir, "archive")
	rows, files, err := archiveCheckpointsToParquet(cpDir, "testcluster", "node001", 2000)
	if err != nil {
		t.Fatal(err)
	}
	if len(files) != 1 || files[0] != "1000.json" {
		t.Fatalf("expected 1 file, got %v", files)
	}

	parquetFile := filepath.Join(archiveDir, "testcluster", "1000.parquet")
	writer, err := newParquetArchiveWriter(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	sortParquetRows(rows)
	if err := writer.WriteHostRows(rows); err != nil {
		t.Fatal(err)
	}
	if err := writer.Close(); err != nil {
		t.Fatal(err)
	}

	// Read back and verify
	f, err := os.Open(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	stat, _ := f.Stat()
	pf, err := pq.OpenFile(f, stat.Size())
	if err != nil {
		t.Fatal(err)
	}

	reader := pq.NewGenericReader[ParquetMetricRow](pf)
	readRows := make([]ParquetMetricRow, 100)
	n, err := reader.Read(readRows)
	if err != nil && n == 0 {
		t.Fatal(err)
	}
	readRows = readRows[:n]
	reader.Close()

	// We expect: cpu_load(3) + mem_used(3) + mem_bw(3) = 9 rows
	if n != 9 {
		t.Fatalf("expected 9 rows in parquet file, got %d", n)
	}

	// Verify cluster and hostname are set correctly
	for _, r := range readRows {
		if r.Cluster != "testcluster" {
			t.Errorf("expected cluster=testcluster, got %s", r.Cluster)
		}
		if r.Hostname != "node001" {
			t.Errorf("expected hostname=node001, got %s", r.Hostname)
		}
	}

	// Verify parquet file is smaller than JSON (compression working)
	if stat.Size() == 0 {
		t.Error("parquet file is empty")
	}

	t.Logf("Parquet file size: %d bytes for %d rows", stat.Size(), n)
}

func TestLoadCheckpointFileFromDisk_JSON(t *testing.T) {
	tmpDir := t.TempDir()

	cf := &CheckpointFile{
		From: 1000,
		To:   1060,
		Metrics: map[string]*CheckpointMetrics{
			"test_metric": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{1.0, 2.0, 3.0},
			},
		},
		Children: make(map[string]*CheckpointFile),
	}

	filename := filepath.Join(tmpDir, "1000.json")
	data, err := json.Marshal(cf)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filename, data, 0o644); err != nil {
		t.Fatal(err)
	}

	loaded, err := loadCheckpointFileFromDisk(filename)
	if err != nil {
		t.Fatal(err)
	}

	if loaded.From != 1000 || loaded.To != 1060 {
		t.Errorf("expected From=1000, To=1060, got From=%d, To=%d", loaded.From, loaded.To)
	}

	m, ok := loaded.Metrics["test_metric"]
	if !ok {
		t.Fatal("expected test_metric in loaded checkpoint")
	}
	if m.Frequency != 60 || m.Start != 1000 || len(m.Data) != 3 {
		t.Errorf("unexpected metric data: %+v", m)
	}
}


Now update the test file to use the new API:


func TestFlattenCheckpointFile(t *testing.T) {
	cf := &CheckpointFile{
		From: 1000,
		To:   1060,
		Metrics: map[string]*CheckpointMetrics{
			"cpu_load": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{0.5, 0.7, schema.NaN},
			},
		},
		Children: map[string]*CheckpointFile{
			"socket0": {
				Metrics: map[string]*CheckpointMetrics{
					"mem_bw": {
						Frequency: 60,
						Start:     1000,
						Data:      []schema.Float{100.0, schema.NaN, 200.0},
					},
				},
				Children: make(map[string]*CheckpointFile),
			},
		},
	}

	rows := flattenCheckpointFile(cf, "fritz", "node001", "node", "", nil)

	// cpu_load: 2 non-NaN values at node scope
	// mem_bw: 2 non-NaN values at socket0 scope
	if len(rows) != 4 {
		t.Fatalf("expected 4 rows, got %d", len(rows))
	}

	// Verify a node-scope row
	found := false
	for _, r := range rows {
		if r.Metric == "cpu_load" && r.Timestamp == 1000 {
			found = true
			if r.Cluster != "fritz" || r.Hostname != "node001" || r.Scope != "node" || r.Value != 0.5 {
				t.Errorf("unexpected row: %+v", r)
			}
		}
	}
	if !found {
		t.Error("expected cpu_load row at timestamp 1000")
	}

	// Verify a socket-scope row
	found = false
	for _, r := range rows {
		if r.Metric == "mem_bw" && r.Scope == "socket" && r.ScopeID == "0" {
			found = true
		}
	}
	if !found {
		t.Error("expected mem_bw row with scope=socket, scope_id=0")
	}
}

func TestWriteCheckpointFile(t *testing.T) {
	cf := &CheckpointFile{
		From: 1000,
		To:   1060,
		Metrics: map[string]*CheckpointMetrics{
			"cpu_load": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{0.5, 0.7, schema.NaN},
			},
		},
		Children: map[string]*CheckpointFile{
			"socket0": {
				Metrics: map[string]*CheckpointMetrics{
					"mem_bw": {
						Frequency: 60,
						Start:     1000,
						Data:      []schema.Float{100.0, schema.NaN, 200.0},
					},
				},
				Children: make(map[string]*CheckpointFile),
			},
		},
	}

	tmpDir := t.TempDir()
	parquetFile := filepath.Join(tmpDir, "test.parquet")
	writer, err := newParquetArchiveWriter(parquetFile)
	if err != nil {
		t.Fatal(err)
	}

	if err := writer.WriteCheckpointFile(cf, "fritz", "node001", "node", ""); err != nil {
		t.Fatal(err)
	}
	if err := writer.Close(); err != nil {
		t.Fatal(err)
	}

	// cpu_load: 2 non-NaN values at node scope
	// mem_bw: 2 non-NaN values at socket0 scope
	if writer.count != 4 {
		t.Fatalf("expected 4 rows written, got %d", writer.count)
	}

	// Read back and verify
	f, err := os.Open(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	stat, _ := f.Stat()
	pf, err := pq.OpenFile(f, stat.Size())
	if err != nil {
		t.Fatal(err)
	}

	reader := pq.NewGenericReader[ParquetMetricRow](pf)
	readRows := make([]ParquetMetricRow, 100)
	n, err := reader.Read(readRows)
	if err != nil && n == 0 {
		t.Fatal(err)
	}
	readRows = readRows[:n]
	reader.Close()

	if n != 4 {
		t.Fatalf("expected 4 rows, got %d", n)
	}

	// Verify a node-scope row
	found := false
	for _, r := range readRows {
		if r.Metric == "cpu_load" && r.Timestamp == 1000 {
			found = true
			if r.Cluster != "fritz" || r.Hostname != "node001" || r.Scope != "node" || r.Value != 0.5 {
				t.Errorf("unexpected row: %+v", r)
			}
		}
	}
	if !found {
		t.Error("expected cpu_load row at timestamp 1000")
	}

	// Verify a socket-scope row
	found = false
	for _, r := range readRows {
		if r.Metric == "mem_bw" && r.Scope == "socket" && r.ScopeID == "0" {
			found = true
		}
	}
	if !found {
		t.Error("expected mem_bw row with scope=socket, scope_id=0")
	}
}


The file /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/parquetArchive_test.go has been updated successfully.

func TestWriteCheckpointFile(t *testing.T) {
	cf := &CheckpointFile{
		From: 1000,
		To:   1060,
		Metrics: map[string]*CheckpointMetrics{
			"cpu_load": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{0.5, 0.7, schema.NaN},
			},
		},
		Children: map[string]*CheckpointFile{
			"socket0": {
				Metrics: map[string]*CheckpointMetrics{
					"mem_bw": {
						Frequency: 60,
						Start:     1000,
						Data:      []schema.Float{100.0, schema.NaN, 200.0},
					},
				},
				Children: make(map[string]*CheckpointFile),
			},
		},
	}

	tmpDir := t.TempDir()
	parquetFile := filepath.Join(tmpDir, "test.parquet")
	writer, err := newParquetArchiveWriter(parquetFile)
	if err != nil {
		t.Fatal(err)
	}

	if err := writer.WriteCheckpointFile(cf, "fritz", "node001", "node", ""); err != nil {
		t.Fatal(err)
	}
	if err := writer.Close(); err != nil {
		t.Fatal(err)
	}

	// cpu_load: 2 non-NaN values at node scope
	// mem_bw: 2 non-NaN values at socket0 scope
	if writer.count != 4 {
		t.Fatalf("expected 4 rows written, got %d", writer.count)
	}

	// Read back and verify
	f, err := os.Open(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	stat, _ := f.Stat()
	pf, err := pq.OpenFile(f, stat.Size())
	if err != nil {
		t.Fatal(err)
	}

	reader := pq.NewGenericReader[ParquetMetricRow](pf)
	readRows := make([]ParquetMetricRow, 100)
	n, err := reader.Read(readRows)
	if err != nil && n == 0 {
		t.Fatal(err)
	}
	readRows = readRows[:n]
	reader.Close()

	if n != 4 {
		t.Fatalf("expected 4 rows, got %d", n)
	}

	// Verify a node-scope row
	found := false
	for _, r := range readRows {
		if r.Metric == "cpu_load" && r.Timestamp == 1000 {
			found = true
			if r.Cluster != "fritz" || r.Hostname != "node001" || r.Scope != "node" || r.Value != 0.5 {
				t.Errorf("unexpected row: %+v", r)
			}
		}
	}
	if !found {
		t.Error("expected cpu_load row at timestamp 1000")
	}

	// Verify a socket-scope row
	found = false
	for _, r := range readRows {
		if r.Metric == "mem_bw" && r.Scope == "socket" && r.ScopeID == "0" {
			found = true
		}
	}
	if !found {
		t.Error("expected mem_bw row with scope=socket, scope_id=0")
	}
}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"encoding/json"
	"os"
	"path/filepath"
	"testing"

	"github.com/ClusterCockpit/cc-lib/v2/schema"
	pq "github.com/parquet-go/parquet-go"
)

func TestParseScopeFromName(t *testing.T) {
	tests := []struct {
		name      string
		wantScope string
		wantID    string
	}{
		{"socket0", "socket", "0"},
		{"socket12", "socket", "12"},
		{"core0", "core", "0"},
		{"core127", "core", "127"},
		{"cpu0", "hwthread", "0"},
		{"hwthread5", "hwthread", "5"},
		{"memoryDomain0", "memoryDomain", "0"},
		{"accelerator0", "accelerator", "0"},
		{"unknown", "unknown", ""},
		{"socketX", "socketX", ""}, // not numeric suffix
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			scope, id := parseScopeFromName(tt.name)
			if scope != tt.wantScope || id != tt.wantID {
				t.Errorf("parseScopeFromName(%q) = (%q, %q), want (%q, %q)",
					tt.name, scope, id, tt.wantScope, tt.wantID)
			}
		})
	}
}

func TestFlattenCheckpointFile(t *testing.T) {
	cf := &CheckpointFile{
		From: 1000,
		To:   1060,
		Metrics: map[string]*CheckpointMetrics{
			"cpu_load": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{0.5, 0.7, schema.NaN},
			},
		},
		Children: map[string]*CheckpointFile{
			"socket0": {
				Metrics: map[string]*CheckpointMetrics{
					"mem_bw": {
						Frequency: 60,
						Start:     1000,
						Data:      []schema.Float{100.0, schema.NaN, 200.0},
					},
				},
				Children: make(map[string]*CheckpointFile),
			},
		},
	}

	rows := flattenCheckpointFile(cf, "fritz", "node001", "node", "", nil)

	// cpu_load: 2 non-NaN values at node scope
	// mem_bw: 2 non-NaN values at socket0 scope
	if len(rows) != 4 {
		t.Fatalf("expected 4 rows, got %d", len(rows))
	}

	// Verify a node-scope row
	found := false
	for _, r := range rows {
		if r.Metric == "cpu_load" && r.Timestamp == 1000 {
			found = true
			if r.Cluster != "fritz" || r.Hostname != "node001" || r.Scope != "node" || r.Value != 0.5 {
				t.Errorf("unexpected row: %+v", r)
			}
		}
	}
	if !found {
		t.Error("expected cpu_load row at timestamp 1000")
	}

	// Verify a socket-scope row
	found = false
	for _, r := range rows {
		if r.Metric == "mem_bw" && r.Scope == "socket" && r.ScopeID == "0" {
			found = true
		}
	}
	if !found {
		t.Error("expected mem_bw row with scope=socket, scope_id=0")
	}
}

func TestParquetArchiveRoundtrip(t *testing.T) {
	tmpDir := t.TempDir()

	// Create checkpoint files on disk (JSON format)
	cpDir := filepath.Join(tmpDir, "checkpoints", "testcluster", "node001")
	if err := os.MkdirAll(cpDir, 0o755); err != nil {
		t.Fatal(err)
	}

	cf := &CheckpointFile{
		From: 1000,
		To:   1180,
		Metrics: map[string]*CheckpointMetrics{
			"cpu_load": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{0.5, 0.7, 0.9},
			},
			"mem_used": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{45.0, 46.0, 47.0},
			},
		},
		Children: map[string]*CheckpointFile{
			"socket0": {
				Metrics: map[string]*CheckpointMetrics{
					"mem_bw": {
						Frequency: 60,
						Start:     1000,
						Data:      []schema.Float{100.0, 110.0, 120.0},
					},
				},
				Children: make(map[string]*CheckpointFile),
			},
		},
	}

	// Write JSON checkpoint
	cpFile := filepath.Join(cpDir, "1000.json")
	data, err := json.Marshal(cf)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(cpFile, data, 0o644); err != nil {
		t.Fatal(err)
	}

	// Archive to Parquet
	archiveDir := filepath.Join(tmpDir, "archive")
	rows, files, err := archiveCheckpointsToParquet(cpDir, "testcluster", "node001", 2000)
	if err != nil {
		t.Fatal(err)
	}
	if len(files) != 1 || files[0] != "1000.json" {
		t.Fatalf("expected 1 file, got %v", files)
	}

	parquetFile := filepath.Join(archiveDir, "testcluster", "1000.parquet")
	writer, err := newParquetArchiveWriter(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	sortParquetRows(rows)
	if err := writer.WriteHostRows(rows); err != nil {
		t.Fatal(err)
	}
	if err := writer.Close(); err != nil {
		t.Fatal(err)
	}

	// Read back and verify
	f, err := os.Open(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	stat, _ := f.Stat()
	pf, err := pq.OpenFile(f, stat.Size())
	if err != nil {
		t.Fatal(err)
	}

	reader := pq.NewGenericReader[ParquetMetricRow](pf)
	readRows := make([]ParquetMetricRow, 100)
	n, err := reader.Read(readRows)
	if err != nil && n == 0 {
		t.Fatal(err)
	}
	readRows = readRows[:n]
	reader.Close()

	// We expect: cpu_load(3) + mem_used(3) + mem_bw(3) = 9 rows
	if n != 9 {
		t.Fatalf("expected 9 rows in parquet file, got %d", n)
	}

	// Verify cluster and hostname are set correctly
	for _, r := range readRows {
		if r.Cluster != "testcluster" {
			t.Errorf("expected cluster=testcluster, got %s", r.Cluster)
		}
		if r.Hostname != "node001" {
			t.Errorf("expected hostname=node001, got %s", r.Hostname)
		}
	}

	// Verify parquet file is smaller than JSON (compression working)
	if stat.Size() == 0 {
		t.Error("parquet file is empty")
	}

	t.Logf("Parquet file size: %d bytes for %d rows", stat.Size(), n)
}

func TestLoadCheckpointFileFromDisk_JSON(t *testing.T) {
	tmpDir := t.TempDir()

	cf := &CheckpointFile{
		From: 1000,
		To:   1060,
		Metrics: map[string]*CheckpointMetrics{
			"test_metric": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{1.0, 2.0, 3.0},
			},
		},
		Children: make(map[string]*CheckpointFile),
	}

	filename := filepath.Join(tmpDir, "1000.json")
	data, err := json.Marshal(cf)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filename, data, 0o644); err != nil {
		t.Fatal(err)
	}

	loaded, err := loadCheckpointFileFromDisk(filename)
	if err != nil {
		t.Fatal(err)
	}

	if loaded.From != 1000 || loaded.To != 1060 {
		t.Errorf("expected From=1000, To=1060, got From=%d, To=%d", loaded.From, loaded.To)
	}

	m, ok := loaded.Metrics["test_metric"]
	if !ok {
		t.Fatal("expected test_metric in loaded checkpoint")
	}
	if m.Frequency != 60 || m.Start != 1000 || len(m.Data) != 3 {
		t.Errorf("unexpected metric data: %+v", m)
	}
}

-func TestFlattenCheckpointFile(t *testing.T) {

+func TestWriteCheckpointFile(t *testing.T) {

-  rows := flattenCheckpointFile(cf, "fritz", "node001", "node", "", nil)

+  parquetFile := filepath.Join(tmpDir, "test.parquet")

+  writer, err := newParquetArchiveWriter(parquetFile)

+  if err := writer.WriteCheckpointFile(cf, "fritz", "node001", "node", ""); err != nil {

+  if err := writer.Close(); err != nil {

-    t.Fatalf("expected 4 rows, got %d", len(rows))

+    t.Fatalf("expected 4 rows written, got %d", writer.count)

+  pf, err := pq.OpenFile(f, stat.Size())

+  reader := pq.NewGenericReader[ParquetMetricRow](pf)

+  readRows := make([]ParquetMetricRow, 100)

+    t.Fatalf("expected 4 rows, got %d", n)


Now update the roundtrip test:


	// Archive to Parquet
	archiveDir := filepath.Join(tmpDir, "archive")
	rows, files, err := archiveCheckpointsToParquet(cpDir, "testcluster", "node001", 2000)
	if err != nil {
		t.Fatal(err)
	}
	if len(files) != 1 || files[0] != "1000.json" {
		t.Fatalf("expected 1 file, got %v", files)
	}

	parquetFile := filepath.Join(archiveDir, "testcluster", "1000.parquet")
	writer, err := newParquetArchiveWriter(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	sortParquetRows(rows)
	if err := writer.WriteHostRows(rows); err != nil {
		t.Fatal(err)
	}
	if err := writer.Close(); err != nil {
		t.Fatal(err)
	}

	// Archive to Parquet
	archiveDir := filepath.Join(tmpDir, "archive")
	checkpoints, files, err := loadCheckpointFiles(cpDir, 2000)
	if err != nil {
		t.Fatal(err)
	}
	if len(files) != 1 || files[0] != "1000.json" {
		t.Fatalf("expected 1 file, got %v", files)
	}

	parquetFile := filepath.Join(archiveDir, "testcluster", "1000.parquet")
	writer, err := newParquetArchiveWriter(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	for _, cp := range checkpoints {
		if err := writer.WriteCheckpointFile(cp, "testcluster", "node001", "node", ""); err != nil {
			t.Fatal(err)
		}
	}
	if err := writer.Close(); err != nil {
		t.Fatal(err)
	}


	// Archive to Parquet
	archiveDir := filepath.Join(tmpDir, "archive")
	checkpoints, files, err := loadCheckpointFiles(cpDir, 2000)
	if err != nil {
		t.Fatal(err)
	}
	if len(files) != 1 || files[0] != "1000.json" {
		t.Fatalf("expected 1 file, got %v", files)
	}

	parquetFile := filepath.Join(archiveDir, "testcluster", "1000.parquet")
	writer, err := newParquetArchiveWriter(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	for _, cp := range checkpoints {
		if err := writer.WriteCheckpointFile(cp, "testcluster", "node001", "node", ""); err != nil {
			t.Fatal(err)
		}
	}
	if err := writer.Close(); err != nil {
		t.Fatal(err)
	}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"encoding/json"
	"os"
	"path/filepath"
	"testing"

	"github.com/ClusterCockpit/cc-lib/v2/schema"
	pq "github.com/parquet-go/parquet-go"
)

func TestParseScopeFromName(t *testing.T) {
	tests := []struct {
		name      string
		wantScope string
		wantID    string
	}{
		{"socket0", "socket", "0"},
		{"socket12", "socket", "12"},
		{"core0", "core", "0"},
		{"core127", "core", "127"},
		{"cpu0", "hwthread", "0"},
		{"hwthread5", "hwthread", "5"},
		{"memoryDomain0", "memoryDomain", "0"},
		{"accelerator0", "accelerator", "0"},
		{"unknown", "unknown", ""},
		{"socketX", "socketX", ""}, // not numeric suffix
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			scope, id := parseScopeFromName(tt.name)
			if scope != tt.wantScope || id != tt.wantID {
				t.Errorf("parseScopeFromName(%q) = (%q, %q), want (%q, %q)",
					tt.name, scope, id, tt.wantScope, tt.wantID)
			}
		})
	}
}

func TestWriteCheckpointFile(t *testing.T) {
	cf := &CheckpointFile{
		From: 1000,
		To:   1060,
		Metrics: map[string]*CheckpointMetrics{
			"cpu_load": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{0.5, 0.7, schema.NaN},
			},
		},
		Children: map[string]*CheckpointFile{
			"socket0": {
				Metrics: map[string]*CheckpointMetrics{
					"mem_bw": {
						Frequency: 60,
						Start:     1000,
						Data:      []schema.Float{100.0, schema.NaN, 200.0},
					},
				},
				Children: make(map[string]*CheckpointFile),
			},
		},
	}

	tmpDir := t.TempDir()
	parquetFile := filepath.Join(tmpDir, "test.parquet")
	writer, err := newParquetArchiveWriter(parquetFile)
	if err != nil {
		t.Fatal(err)
	}

	if err := writer.WriteCheckpointFile(cf, "fritz", "node001", "node", ""); err != nil {
		t.Fatal(err)
	}
	if err := writer.Close(); err != nil {
		t.Fatal(err)
	}

	// cpu_load: 2 non-NaN values at node scope
	// mem_bw: 2 non-NaN values at socket0 scope
	if writer.count != 4 {
		t.Fatalf("expected 4 rows written, got %d", writer.count)
	}

	// Read back and verify
	f, err := os.Open(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	stat, _ := f.Stat()
	pf, err := pq.OpenFile(f, stat.Size())
	if err != nil {
		t.Fatal(err)
	}

	reader := pq.NewGenericReader[ParquetMetricRow](pf)
	readRows := make([]ParquetMetricRow, 100)
	n, err := reader.Read(readRows)
	if err != nil && n == 0 {
		t.Fatal(err)
	}
	readRows = readRows[:n]
	reader.Close()

	if n != 4 {
		t.Fatalf("expected 4 rows, got %d", n)
	}

	// Verify a node-scope row
	found := false
	for _, r := range readRows {
		if r.Metric == "cpu_load" && r.Timestamp == 1000 {
			found = true
			if r.Cluster != "fritz" || r.Hostname != "node001" || r.Scope != "node" || r.Value != 0.5 {
				t.Errorf("unexpected row: %+v", r)
			}
		}
	}
	if !found {
		t.Error("expected cpu_load row at timestamp 1000")
	}

	// Verify a socket-scope row
	found = false
	for _, r := range readRows {
		if r.Metric == "mem_bw" && r.Scope == "socket" && r.ScopeID == "0" {
			found = true
		}
	}
	if !found {
		t.Error("expected mem_bw row with scope=socket, scope_id=0")
	}
}

func TestParquetArchiveRoundtrip(t *testing.T) {
	tmpDir := t.TempDir()

	// Create checkpoint files on disk (JSON format)
	cpDir := filepath.Join(tmpDir, "checkpoints", "testcluster", "node001")
	if err := os.MkdirAll(cpDir, 0o755); err != nil {
		t.Fatal(err)
	}

	cf := &CheckpointFile{
		From: 1000,
		To:   1180,
		Metrics: map[string]*CheckpointMetrics{
			"cpu_load": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{0.5, 0.7, 0.9},
			},
			"mem_used": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{45.0, 46.0, 47.0},
			},
		},
		Children: map[string]*CheckpointFile{
			"socket0": {
				Metrics: map[string]*CheckpointMetrics{
					"mem_bw": {
						Frequency: 60,
						Start:     1000,
						Data:      []schema.Float{100.0, 110.0, 120.0},
					},
				},
				Children: make(map[string]*CheckpointFile),
			},
		},
	}

	// Write JSON checkpoint
	cpFile := filepath.Join(cpDir, "1000.json")
	data, err := json.Marshal(cf)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(cpFile, data, 0o644); err != nil {
		t.Fatal(err)
	}

	// Archive to Parquet
	archiveDir := filepath.Join(tmpDir, "archive")
	rows, files, err := archiveCheckpointsToParquet(cpDir, "testcluster", "node001", 2000)
	if err != nil {
		t.Fatal(err)
	}
	if len(files) != 1 || files[0] != "1000.json" {
		t.Fatalf("expected 1 file, got %v", files)
	}

	parquetFile := filepath.Join(archiveDir, "testcluster", "1000.parquet")
	writer, err := newParquetArchiveWriter(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	sortParquetRows(rows)
	if err := writer.WriteHostRows(rows); err != nil {
		t.Fatal(err)
	}
	if err := writer.Close(); err != nil {
		t.Fatal(err)
	}

	// Read back and verify
	f, err := os.Open(parquetFile)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	stat, _ := f.Stat()
	pf, err := pq.OpenFile(f, stat.Size())
	if err != nil {
		t.Fatal(err)
	}

	reader := pq.NewGenericReader[ParquetMetricRow](pf)
	readRows := make([]ParquetMetricRow, 100)
	n, err := reader.Read(readRows)
	if err != nil && n == 0 {
		t.Fatal(err)
	}
	readRows = readRows[:n]
	reader.Close()

	// We expect: cpu_load(3) + mem_used(3) + mem_bw(3) = 9 rows
	if n != 9 {
		t.Fatalf("expected 9 rows in parquet file, got %d", n)
	}

	// Verify cluster and hostname are set correctly
	for _, r := range readRows {
		if r.Cluster != "testcluster" {
			t.Errorf("expected cluster=testcluster, got %s", r.Cluster)
		}
		if r.Hostname != "node001" {
			t.Errorf("expected hostname=node001, got %s", r.Hostname)
		}
	}

	// Verify parquet file is smaller than JSON (compression working)
	if stat.Size() == 0 {
		t.Error("parquet file is empty")
	}

	t.Logf("Parquet file size: %d bytes for %d rows", stat.Size(), n)
}

func TestLoadCheckpointFileFromDisk_JSON(t *testing.T) {
	tmpDir := t.TempDir()

	cf := &CheckpointFile{
		From: 1000,
		To:   1060,
		Metrics: map[string]*CheckpointMetrics{
			"test_metric": {
				Frequency: 60,
				Start:     1000,
				Data:      []schema.Float{1.0, 2.0, 3.0},
			},
		},
		Children: make(map[string]*CheckpointFile),
	}

	filename := filepath.Join(tmpDir, "1000.json")
	data, err := json.Marshal(cf)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filename, data, 0o644); err != nil {
		t.Fatal(err)
	}

	loaded, err := loadCheckpointFileFromDisk(filename)
	if err != nil {
		t.Fatal(err)
	}

	if loaded.From != 1000 || loaded.To != 1060 {
		t.Errorf("expected From=1000, To=1060, got From=%d, To=%d", loaded.From, loaded.To)
	}

	m, ok := loaded.Metrics["test_metric"]
	if !ok {
		t.Fatal("expected test_metric in loaded checkpoint")
	}
	if m.Frequency != 60 || m.Start != 1000 || len(m.Data) != 3 {
		t.Errorf("unexpected metric data: %+v", m)
	}
}

-  rows, files, err := archiveCheckpointsToParquet(cpDir, "testcluster", "node001", 2000)

+  checkpoints, files, err := loadCheckpointFiles(cpDir, 2000)

-  if err := writer.WriteHostRows(rows); err != nil {

+    if err := writer.WriteCheckpointFile(cp, "testcluster", "node001", "node", ""); err != nil {


The diagnostics are stale — they're still showing old line numbers. Let me also check that the `schema` import is still needed and remove unused imports.


No matches found


Good, all old references are gone. Now check if the unused `cclog` import needs to be removed from parquetArchive.go:


Good, I already removed the `cclog` import in my edit. Now let's build and test:


Bash

Build metricstore package


(Bash completed with no output)


go test ./pkg/metricstore/... -run "TestParseScopeFromName|TestWriteCheckpointFile|TestParquetArchiveRoundtrip|TestLoadCheckpointFileFromDisk" -v 2>&1

Run parquet-related tests