     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package parquet
     7→
     8→import (
     9→	"bytes"
    10→	"context"
    11→	"encoding/json"
    12→	"fmt"
    13→	"io"
    14→	"os"
    15→	"path/filepath"
    16→	"strings"
    17→
    18→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    19→	"github.com/aws/aws-sdk-go-v2/aws"
    20→	awsconfig "github.com/aws/aws-sdk-go-v2/config"
    21→	"github.com/aws/aws-sdk-go-v2/credentials"
    22→	"github.com/aws/aws-sdk-go-v2/service/s3"
    23→	pq "github.com/parquet-go/parquet-go"
    24→)
    25→
    26→// ReadParquetFile reads all ParquetJobRow entries from parquet-encoded bytes.
    27→func ReadParquetFile(data []byte) ([]ParquetJobRow, error) {
    28→	file, err := pq.OpenFile(bytes.NewReader(data), int64(len(data)))
    29→	if err != nil {
    30→		return nil, fmt.Errorf("open parquet: %w", err)
    31→	}
    32→
    33→	reader := pq.NewGenericReader[ParquetJobRow](file)
    34→	defer reader.Close()
    35→
    36→	numRows := file.NumRows()
    37→	rows := make([]ParquetJobRow, numRows)
    38→	n, err := reader.Read(rows)
    39→	if err != nil && err != io.EOF {
    40→		return nil, fmt.Errorf("read parquet rows: %w", err)
    41→	}
    42→
    43→	return rows[:n], nil
    44→}
    45→
    46→// ParquetSource abstracts reading parquet archives from different storage backends.
    47→type ParquetSource interface {
    48→	GetClusters() ([]string, error)
    49→	ListParquetFiles(cluster string) ([]string, error)
    50→	ReadFile(path string) ([]byte, error)
    51→	ReadClusterConfig(cluster string) (*schema.Cluster, error)
    52→}
    53→
    54→// FileParquetSource reads parquet archives from a local filesystem directory.
    55→type FileParquetSource struct {
    56→	path string
    57→}
    58→
    59→func NewFileParquetSource(path string) *FileParquetSource {
    60→	return &FileParquetSource{path: path}
    61→}
    62→
    63→func (fs *FileParquetSource) GetClusters() ([]string, error) {
    64→	entries, err := os.ReadDir(fs.path)
    65→	if err != nil {
    66→		return nil, fmt.Errorf("read directory: %w", err)
    67→	}
    68→
    69→	var clusters []string
    70→	for _, e := range entries {
    71→		if e.IsDir() {
    72→			clusters = append(clusters, e.Name())
    73→		}
    74→	}
    75→	return clusters, nil
    76→}
    77→
    78→func (fs *FileParquetSource) ListParquetFiles(cluster string) ([]string, error) {
    79→	dir := filepath.Join(fs.path, cluster)
    80→	entries, err := os.ReadDir(dir)
    81→	if err != nil {
    82→		return nil, fmt.Errorf("read cluster directory: %w", err)
    83→	}
    84→
    85→	var files []string
    86→	for _, e := range entries {
    87→		if !e.IsDir() && strings.HasSuffix(e.Name(), ".parquet") {
    88→			files = append(files, filepath.Join(cluster, e.Name()))
    89→		}
    90→	}
    91→	return files, nil
    92→}
    93→
    94→func (fs *FileParquetSource) ReadFile(path string) ([]byte, error) {
    95→	return os.ReadFile(filepath.Join(fs.path, path))
    96→}
    97→
    98→func (fs *FileParquetSource) ReadClusterConfig(cluster string) (*schema.Cluster, error) {
    99→	data, err := os.ReadFile(filepath.Join(fs.path, cluster, "cluster.json"))
   100→	if err != nil {
   101→		return nil, fmt.Errorf("read cluster.json: %w", err)
   102→	}
   103→	var cfg schema.Cluster
   104→	if err := json.Unmarshal(data, &cfg); err != nil {
   105→		return nil, fmt.Errorf("unmarshal cluster config: %w", err)
   106→	}
   107→	return &cfg, nil
   108→}
   109→
   110→// S3ParquetSource reads parquet archives from an S3-compatible object store.
   111→type S3ParquetSource struct {
   112→	client *s3.Client
   113→	bucket string
   114→}
   115→
   116→func NewS3ParquetSource(cfg S3TargetConfig) (*S3ParquetSource, error) {
   117→	if cfg.Bucket == "" {
   118→		return nil, fmt.Errorf("S3 source: empty bucket name")
   119→	}
   120→
   121→	region := cfg.Region
   122→	if region == "" {
   123→		region = "us-east-1"
   124→	}
   125→
   126→	awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(),
   127→		awsconfig.WithRegion(region),
   128→		awsconfig.WithCredentialsProvider(
   129→			credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, ""),
   130→		),
   131→	)
   132→	if err != nil {
   133→		return nil, fmt.Errorf("S3 source: load AWS config: %w", err)
   134→	}
   135→
   136→	opts := func(o *s3.Options) {
   137→		if cfg.Endpoint != "" {
   138→			o.BaseEndpoint = aws.String(cfg.Endpoint)
   139→		}
   140→		o.UsePathStyle = cfg.UsePathStyle
   141→	}
   142→
   143→	client := s3.NewFromConfig(awsCfg, opts)
   144→	return &S3ParquetSource{client: client, bucket: cfg.Bucket}, nil
   145→}
   146→
   147→func (ss *S3ParquetSource) GetClusters() ([]string, error) {
   148→	ctx := context.Background()
   149→	paginator := s3.NewListObjectsV2Paginator(ss.client, &s3.ListObjectsV2Input{
   150→		Bucket:    aws.String(ss.bucket),
   151→		Delimiter: aws.String("/"),
   152→	})
   153→
   154→	var clusters []string
   155→	for paginator.HasMorePages() {
   156→		page, err := paginator.NextPage(ctx)
   157→		if err != nil {
   158→			return nil, fmt.Errorf("S3 source: list clusters: %w", err)
   159→		}
   160→		for _, prefix := range page.CommonPrefixes {
   161→			if prefix.Prefix != nil {
   162→				name := strings.TrimSuffix(*prefix.Prefix, "/")
   163→				clusters = append(clusters, name)
   164→			}
   165→		}
   166→	}
   167→	return clusters, nil
   168→}
   169→
   170→func (ss *S3ParquetSource) ListParquetFiles(cluster string) ([]string, error) {
   171→	ctx := context.Background()
   172→	prefix := cluster + "/"
   173→	paginator := s3.NewListObjectsV2Paginator(ss.client, &s3.ListObjectsV2Input{
   174→		Bucket: aws.String(ss.bucket),
   175→		Prefix: aws.String(prefix),
   176→	})
   177→
   178→	var files []string
   179→	for paginator.HasMorePages() {
   180→		page, err := paginator.NextPage(ctx)
   181→		if err != nil {
   182→			return nil, fmt.Errorf("S3 source: list parquet files: %w", err)
   183→		}
   184→		for _, obj := range page.Contents {
   185→			if obj.Key != nil && strings.HasSuffix(*obj.Key, ".parquet") {
   186→				files = append(files, *obj.Key)
   187→			}
   188→		}
   189→	}
   190→	return files, nil
   191→}
   192→
   193→func (ss *S3ParquetSource) ReadFile(path string) ([]byte, error) {
   194→	ctx := context.Background()
   195→	result, err := ss.client.GetObject(ctx, &s3.GetObjectInput{
   196→		Bucket: aws.String(ss.bucket),
   197→		Key:    aws.String(path),
   198→	})
   199→	if err != nil {
   200→		return nil, fmt.Errorf("S3 source: get object %q: %w", path, err)
   201→	}
   202→	defer result.Body.Close()
   203→	return io.ReadAll(result.Body)
   204→}
   205→
   206→func (ss *S3ParquetSource) ReadClusterConfig(cluster string) (*schema.Cluster, error) {
   207→	data, err := ss.ReadFile(cluster + "/cluster.json")
   208→	if err != nil {
   209→		return nil, fmt.Errorf("read cluster.json: %w", err)
   210→	}
   211→	var cfg schema.Cluster
   212→	if err := json.Unmarshal(data, &cfg); err != nil {
   213→		return nil, fmt.Errorf("unmarshal cluster config: %w", err)
   214→	}
   215→	return &cfg, nil
   216→}
   217→


Bash

grep -n "WriteArchive\|Flush\|writer.Write" /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/*.go | head -30


14 matches in 6 files:

/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go (1):
  273: if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {

/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go (1):
  358: return bw.Flush()

/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/debug.go (1):
  111: return w.Flush()

/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/parquetArchive.go (5):
  106: // Flush remaining batch
  108: if _, err := w.writer.Write(w.batch); err != nil {
  115: if err := w.writer.Flush(); err != nil {
  149: w.writer.Write(w.batch)
  178: if err := w.bw.Flush(); err != nil {

/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/parquetArchive_test.go (2):
  79: if err := writer.WriteCheckpointFile(cf, "fritz", "node001", "node", ""); err != nil {
  208: if err := writer.WriteCheckpointFile(cp, "testcluster", "node001", "node", ""); err != nil {

/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go (4):
  173: ws.w.Flush()
  230: ws.w.Flush()
  244: ws.w.Flush()
  776: if err := bw.Flush(); err != nil {


174:func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
273:					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"context"
    10→	"errors"
    11→	"fmt"
    12→	"os"
    13→	"path/filepath"
    14→	"sync"
    15→	"sync/atomic"
    16→	"time"
    17→
    18→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    19→)
    20→
    21→// Worker for either Archiving or Deleting files
    22→
    23→func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
    24→	if Keys.Cleanup.Mode == "archive" {
    25→		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
    26→		// Run as Archiver
    27→		cleanUpWorker(wg, ctx,
    28→			Keys.RetentionInMemory,
    29→			"archiving",
    30→			Keys.Cleanup.RootDir,
    31→			false,
    32→		)
    33→	} else {
    34→		// Run as Deleter
    35→		cleanUpWorker(wg, ctx,
    36→			Keys.RetentionInMemory,
    37→			"deleting",
    38→			"",
    39→			true,
    40→		)
    41→	}
    42→}
    43→
    44→// cleanUpWorker takes simple values to configure what it does
    45→func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
    46→	wg.Go(func() {
    47→		d, err := time.ParseDuration(interval)
    48→		if err != nil {
    49→			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
    50→		}
    51→		if d <= 0 {
    52→			return
    53→		}
    54→
    55→		ticker := time.NewTicker(d)
    56→		defer ticker.Stop()
    57→
    58→		for {
    59→			select {
    60→			case <-ctx.Done():
    61→				return
    62→			case <-ticker.C:
    63→				t := time.Now().Add(-d)
    64→				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))
    65→
    66→				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)
    67→
    68→				if err != nil {
    69→					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
    70→				} else {
    71→					if delete {
    72→						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
    73→					} else {
    74→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
    75→					}
    76→				}
    77→			}
    78→		}
    79→	})
    80→}
    81→
    82→var ErrNoNewArchiveData error = errors.New("all data already archived")
    83→
    84→// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
    85→// When archiving, consolidates all hosts per cluster into a single Parquet file.
    86→func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
    87→	if deleteInstead {
    88→		return deleteCheckpoints(checkpointsDir, from)
    89→	}
    90→
    91→	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
    92→}
    93→
    94→// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
    95→func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
    96→	entries1, err := os.ReadDir(checkpointsDir)
    97→	if err != nil {
    98→		return 0, err
    99→	}
   100→
   101→	type workItem struct {
   102→		dir           string
   103→		cluster, host string
   104→	}
   105→
   106→	var wg sync.WaitGroup
   107→	n, errs := int32(0), int32(0)
   108→	work := make(chan workItem, Keys.NumWorkers)
   109→
   110→	wg.Add(Keys.NumWorkers)
   111→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   112→		go func() {
   113→			defer wg.Done()
   114→			for item := range work {
   115→				entries, err := os.ReadDir(item.dir)
   116→				if err != nil {
   117→					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
   118→					atomic.AddInt32(&errs, 1)
   119→					continue
   120→				}
   121→
   122→				files, err := findFiles(entries, from, false)
   123→				if err != nil {
   124→					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
   125→					atomic.AddInt32(&errs, 1)
   126→					continue
   127→				}
   128→
   129→				for _, checkpoint := range files {
   130→					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
   131→						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
   132→						atomic.AddInt32(&errs, 1)
   133→					} else {
   134→						atomic.AddInt32(&n, 1)
   135→					}
   136→				}
   137→			}
   138→		}()
   139→	}
   140→
   141→	for _, de1 := range entries1 {
   142→		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
   143→		if e != nil {
   144→			err = e
   145→			continue
   146→		}
   147→
   148→		for _, de2 := range entries2 {
   149→			work <- workItem{
   150→				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
   151→				cluster: de1.Name(),
   152→				host:    de2.Name(),
   153→			}
   154→		}
   155→	}
   156→
   157→	close(work)
   158→	wg.Wait()
   159→
   160→	if err != nil {
   161→		return int(n), err
   162→	}
   163→	if errs > 0 {
   164→		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
   165→	}
   166→	return int(n), nil
   167→}
   168→
   169→// archiveCheckpoints archives checkpoint files to Parquet format.
   170→// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
   171→// Workers load checkpoint files from disk and send CheckpointFile trees on a
   172→// back-pressured channel. The main thread streams each tree directly to Parquet
   173→// rows without materializing all rows in memory.
   174→func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
   175→	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
   176→	startTime := time.Now()
   177→
   178→	clusterEntries, err := os.ReadDir(checkpointsDir)
   179→	if err != nil {
   180→		return 0, err
   181→	}
   182→
   183→	totalFiles := 0
   184→
   185→	for _, clusterEntry := range clusterEntries {
   186→		if !clusterEntry.IsDir() {
   187→			continue
   188→		}
   189→
   190→		cluster := clusterEntry.Name()
   191→		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
   192→		if err != nil {
   193→			return totalFiles, err
   194→		}
   195→
   196→		// Workers load checkpoint files from disk; main thread writes to parquet.
   197→		type hostResult struct {
   198→			checkpoints []*CheckpointFile
   199→			hostname    string
   200→			files       []string // checkpoint filenames to delete after successful write
   201→			dir         string   // checkpoint directory for this host
   202→		}
   203→
   204→		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
   205→		results := make(chan hostResult, 2)
   206→		work := make(chan struct {
   207→			dir, host string
   208→		}, Keys.NumWorkers)
   209→
   210→		var wg sync.WaitGroup
   211→		errs := int32(0)
   212→
   213→		wg.Add(Keys.NumWorkers)
   214→		for w := 0; w < Keys.NumWorkers; w++ {
   215→			go func() {
   216→				defer wg.Done()
   217→				for item := range work {
   218→					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
   219→					if err != nil {
   220→						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
   221→						atomic.AddInt32(&errs, 1)
   222→						continue
   223→					}
   224→					if len(checkpoints) > 0 {
   225→						results <- hostResult{
   226→							checkpoints: checkpoints,
   227→							hostname:    item.host,
   228→							files:       files,
   229→							dir:         item.dir,
   230→						}
   231→					}
   232→				}
   233→			}()
   234→		}
   235→
   236→		go func() {
   237→			for _, hostEntry := range hostEntries {
   238→				if !hostEntry.IsDir() {
   239→					continue
   240→				}
   241→				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
   242→				work <- struct {
   243→					dir, host string
   244→				}{dir: dir, host: hostEntry.Name()}
   245→			}
   246→			close(work)
   247→			wg.Wait()
   248→			close(results)
   249→		}()
   250→
   251→		// Open streaming writer and write each host's checkpoint files as a row group
   252→		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
   253→		writer, err := newParquetArchiveWriter(parquetFile)
   254→		if err != nil {
   255→			// Drain results channel to unblock workers
   256→			for range results {
   257→			}
   258→			return totalFiles, fmt.Errorf("creating parquet writer for cluster %s: %w", cluster, err)
   259→		}
   260→
   261→		type deleteItem struct {
   262→			dir   string
   263→			files []string
   264→		}
   265→		var toDelete []deleteItem
   266→		writeErr := error(nil)
   267→
   268→		for r := range results {
   269→			if writeErr == nil {
   270→				// Stream each checkpoint file directly to parquet rows.
   271→				// Each checkpoint is processed and discarded before the next.
   272→				for _, cf := range r.checkpoints {
   273→					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
   274→						writeErr = err
   275→						break
   276→					}
   277→				}
   278→			}
   279→			// Always track files for deletion (even if write failed, we still drain)
   280→			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
   281→		}
   282→
   283→		if err := writer.Close(); err != nil && writeErr == nil {
   284→			writeErr = err
   285→		}
   286→
   287→		if errs > 0 {
   288→			return totalFiles, fmt.Errorf("%d errors reading checkpoints for cluster %s", errs, cluster)
   289→		}
   290→
   291→		if writer.count == 0 {
   292→			// No data written — remove empty file
   293→			os.Remove(parquetFile)
   294→			continue
   295→		}
   296→
   297→		if writeErr != nil {
   298→			os.Remove(parquetFile)
   299→			return totalFiles, fmt.Errorf("writing parquet archive for cluster %s: %w", cluster, writeErr)
   300→		}
   301→
   302→		// Delete archived checkpoint files
   303→		for _, item := range toDelete {
   304→			for _, file := range item.files {
   305→				filename := filepath.Join(item.dir, file)
   306→				if err := os.Remove(filename); err != nil {
   307→					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
   308→				} else {
   309→					totalFiles++
   310→				}
   311→			}
   312→		}
   313→
   314→		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
   315→			writer.count, totalFiles, cluster, parquetFile)
   316→	}
   317→
   318→	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
   319→	return totalFiles, nil
   320→}
   321→


Perfect! Now I have a complete understanding. Let me create a comprehensive summary document for the user:

## Summary: Parquet Archive Writing Investigation

I've thoroughly investigated the parquet writing implementation in ClusterCockpit. Here's what I found regarding row group creation and the 32767 limit issue:

### **1. Parquet Writer Implementation**

#### **Two Main Parquet Writers:**

1. **`pkg/archive/parquet/writer.go` - ParquetWriter** (for job metadata)
   - Batches `ParquetJobRow` objects in memory
   - Flushes to disk when batch size exceeds `maxSizeBytes` (configurable in MB)
   - **Creates ONE parquet file per flush** with sorting applied
   - Used in: retention copy/move services and archive-manager conversion tool

2. **`pkg/metricstore/parquetArchive.go` - parquetArchiveWriter** (for metric data)
   - Streams metric rows directly during archiving
   - **Creates ONE row group per checkpoint file** (via `WriteCheckpointFile()` + `writer.Flush()`)
   - Used in: metric store cleanup/archiving (converts checkpoints to parquet)

#### **Row Group Creation Mechanism:**

The issue is in `parquetArchive.go`. Here's how row groups are created:

```go
// Line 103: Called once per checkpoint file
func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {
    w.writeLevel(cf, cluster, hostname, scope, scopeID)

    // Flush remaining batch
    if len(w.batch) > 0 {
        w.writer.Write(w.batch)
    }

    // THIS CREATES A ROW GROUP
    if err := w.writer.Flush(); err != nil {
        return fmt.Errorf("flushing parquet row group: %w", err)
    }
    return nil
}
```

**Critical Problem:** Each call to `writer.Flush()` creates a new row group in the same parquet file. The parquet-go library has a **hard limit of 32767 row groups per file**.

### **2. How the 32767 Limit is Reached**

The `archiveCheckpoints()` function (line 174-320 in `archive.go`):

1. Creates ONE parquet writer per cluster
2. For EACH host in the cluster, loads ALL checkpoint files
3. For EACH checkpoint file, calls `writer.WriteCheckpointFile()` which:
   - Writes metric data to a batch buffer
   - Calls `writer.Flush()` to finalize - **this creates a new row group**

**Example:** If a single cluster has:
- 100 hosts
- Each with 500 checkpoint files
- Total: 100 × 500 = 50,000 flushes = 50,000 row groups

This **exceeds the 32767 limit**, causing the error.

### **3. Key Code Locations**

**Metric archiving flow:**
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go:174-320` - `archiveCheckpoints()`
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/parquetArchive.go:63-184` - `parquetArchiveWriter` implementation

**Parquet library used:**
- `github.com/parquet-go/parquet-go v0.29.0`
- The `SortingWriterConfig` is used for job archives, but the metric archive writer doesn't use it

**Job metadata archiving flow:**
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/archive/parquet/writer.go:20-106` - `ParquetWriter` implementation
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/retentionService.go:107-149` - `transferJobsParquet()`

### **4. Buffering/Batching Logic**

**Metric archive:**
- Batch size: `1024` rows (constant, line 73 in `parquetArchive.go`)
- Writes batches incrementally during metric traversal
- **But flushes row group after EVERY checkpoint file** - this is the bottleneck

**Job archive:**
- Batch size: configurable (default 512 MB, command-line: `-max-file-size`)
- Size estimated based on JSON fields and metadata
- **Creates NEW FILES when batch exceeds max size** (not row groups in same file)
- Much safer approach

### **5. Root Cause Summary**

The 32767 row group limit is reached in the **metric store archiving** (not job archiving) because:

1. **One file per cluster** - all hosts' metrics go to a single parquet file
2. **One row group per checkpoint** - every checkpoint triggers a `Flush()` call
3. **Many checkpoints** - large clusters with many hosts accumulate thousands of checkpoints
4. Result: Single parquet file accumulates too many row groups

### **File Paths for Reference**

- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/parquetArchive.go` - problematic writer
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go` - archiving orchestration
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/archive/parquet/writer.go` - safer job writer (uses separate files)
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/taskmanager/retentionService.go` - retention service integration

agentId: a364ab7c51b6f5a98 (use SendMessage with to: 'a364ab7c51b6f5a98' to continue this agent)
<usage>total_tokens: 59464
tool_uses: 22
duration_ms: 44506</usage>

Prompt:
Investigate how parquet archives are written in this codebase. I need to understand:

1. The parquet writer implementation - how row groups are created and flushed
2. How metric data is structured into parquet rows/row groups
3. Any buffering or batching logic for parquet writes

Key areas to explore:
- `pkg/archive/parquet/` directory - the parquet writer implementation
- Any parquet-related code in `pkg/metricstore/`
- Look for how `WriteArchive`, `NewWriter`, row group flushing, etc. work

The error is: "the limit of 32767 row groups has been reached" - so I need to understand what triggers new row groups and why there could be so many.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"bufio"
    10→	"encoding/binary"
    11→	"encoding/json"
    12→	"fmt"
    13→	"os"
    14→	"path/filepath"
    15→	"sort"
    16→
    17→	pq "github.com/parquet-go/parquet-go"
    18→)
    19→
    20→// ParquetMetricRow is the long-format schema for archived metric data.
    21→// One row per (host, metric, scope, scope_id, timestamp) data point.
    22→// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
    23→type ParquetMetricRow struct {
    24→	Cluster   string  `parquet:"cluster"`
    25→	Hostname  string  `parquet:"hostname"`
    26→	Metric    string  `parquet:"metric"`
    27→	Scope     string  `parquet:"scope"`
    28→	ScopeID   string  `parquet:"scope_id"`
    29→	Timestamp int64   `parquet:"timestamp"`
    30→	Frequency int64   `parquet:"frequency"`
    31→	Value     float32 `parquet:"value"`
    32→}
    33→
    34→// parseScopeFromName infers scope and scope_id from a child level name.
    35→// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
    36→// "a0" (accelerator) → ("accelerator", "0").
    37→// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
    38→func parseScopeFromName(name string) (string, string) {
    39→	prefixes := []struct {
    40→		prefix string
    41→		scope  string
    42→	}{
    43→		{"socket", "socket"},
    44→		{"memoryDomain", "memoryDomain"},
    45→		{"core", "core"},
    46→		{"hwthread", "hwthread"},
    47→		{"cpu", "hwthread"},
    48→		{"accelerator", "accelerator"},
    49→	}
    50→
    51→	for _, p := range prefixes {
    52→		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
    53→			id := name[len(p.prefix):]
    54→			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
    55→				return p.scope, id
    56→			}
    57→		}
    58→	}
    59→
    60→	return name, ""
    61→}
    62→
    63→// parquetArchiveWriter supports incremental writes to a Parquet file.
    64→// Uses streaming writes to avoid accumulating all rows in memory.
    65→type parquetArchiveWriter struct {
    66→	writer *pq.GenericWriter[ParquetMetricRow]
    67→	bw     *bufio.Writer
    68→	f      *os.File
    69→	batch  []ParquetMetricRow // reusable batch buffer
    70→	count  int
    71→}
    72→
    73→const parquetBatchSize = 1024
    74→
    75→// newParquetArchiveWriter creates a streaming Parquet writer with Zstd compression.
    76→func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
    77→	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
    78→		return nil, fmt.Errorf("creating archive directory: %w", err)
    79→	}
    80→
    81→	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
    82→	if err != nil {
    83→		return nil, fmt.Errorf("creating parquet file: %w", err)
    84→	}
    85→
    86→	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer
    87→
    88→	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
    89→		pq.Compression(&pq.Zstd),
    90→	)
    91→
    92→	return &parquetArchiveWriter{
    93→		writer: writer,
    94→		bw:     bw,
    95→		f:      f,
    96→		batch:  make([]ParquetMetricRow, 0, parquetBatchSize),
    97→	}, nil
    98→}
    99→
   100→// WriteCheckpointFile streams a CheckpointFile tree directly to Parquet rows,
   101→// writing metrics in sorted order without materializing all rows in memory.
   102→// Produces one row group per call (typically one host's data).
   103→func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {
   104→	w.writeLevel(cf, cluster, hostname, scope, scopeID)
   105→
   106→	// Flush remaining batch
   107→	if len(w.batch) > 0 {
   108→		if _, err := w.writer.Write(w.batch); err != nil {
   109→			return fmt.Errorf("writing parquet rows: %w", err)
   110→		}
   111→		w.count += len(w.batch)
   112→		w.batch = w.batch[:0]
   113→	}
   114→
   115→	if err := w.writer.Flush(); err != nil {
   116→		return fmt.Errorf("flushing parquet row group: %w", err)
   117→	}
   118→
   119→	return nil
   120→}
   121→
   122→// writeLevel recursively writes metrics from a CheckpointFile level.
   123→// Metric names and child names are sorted for deterministic, compression-friendly output.
   124→func (w *parquetArchiveWriter) writeLevel(cf *CheckpointFile, cluster, hostname, scope, scopeID string) {
   125→	// Sort metric names for deterministic order
   126→	metricNames := make([]string, 0, len(cf.Metrics))
   127→	for name := range cf.Metrics {
   128→		metricNames = append(metricNames, name)
   129→	}
   130→	sort.Strings(metricNames)
   131→
   132→	for _, metricName := range metricNames {
   133→		cm := cf.Metrics[metricName]
   134→		ts := cm.Start
   135→		for _, v := range cm.Data {
   136→			if !v.IsNaN() {
   137→				w.batch = append(w.batch, ParquetMetricRow{
   138→					Cluster:   cluster,
   139→					Hostname:  hostname,
   140→					Metric:    metricName,
   141→					Scope:     scope,
   142→					ScopeID:   scopeID,
   143→					Timestamp: ts,
   144→					Frequency: cm.Frequency,
   145→					Value:     float32(v),
   146→				})
   147→
   148→				if len(w.batch) >= parquetBatchSize {
   149→					w.writer.Write(w.batch)
   150→					w.count += len(w.batch)
   151→					w.batch = w.batch[:0]
   152→				}
   153→			}
   154→			ts += cm.Frequency
   155→		}
   156→	}
   157→
   158→	// Sort child names for deterministic order
   159→	childNames := make([]string, 0, len(cf.Children))
   160→	for name := range cf.Children {
   161→		childNames = append(childNames, name)
   162→	}
   163→	sort.Strings(childNames)
   164→
   165→	for _, childName := range childNames {
   166→		childScope, childScopeID := parseScopeFromName(childName)
   167→		w.writeLevel(cf.Children[childName], cluster, hostname, childScope, childScopeID)
   168→	}
   169→}
   170→
   171→// Close finalises the Parquet file (footer, buffered I/O, file handle).
   172→func (w *parquetArchiveWriter) Close() error {
   173→	if err := w.writer.Close(); err != nil {
   174→		w.f.Close()
   175→		return fmt.Errorf("closing parquet writer: %w", err)
   176→	}
   177→
   178→	if err := w.bw.Flush(); err != nil {
   179→		w.f.Close()
   180→		return fmt.Errorf("flushing parquet file: %w", err)
   181→	}
   182→
   183→	return w.f.Close()
   184→}
   185→
   186→// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
   187→// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
   188→// before converting it to Parquet format.
   189→func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
   190→	f, err := os.Open(filename)
   191→	if err != nil {
   192→		return nil, err
   193→	}
   194→	defer f.Close()
   195→
   196→	ext := filepath.Ext(filename)
   197→	switch ext {
   198→	case ".json":
   199→		cf := &CheckpointFile{}
   200→		br := bufio.NewReader(f)
   201→		if err := json.NewDecoder(br).Decode(cf); err != nil {
   202→			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
   203→		}
   204→		return cf, nil
   205→
   206→	case ".bin":
   207→		br := bufio.NewReader(f)
   208→		var magic uint32
   209→		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
   210→			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
   211→		}
   212→		if magic != snapFileMagic {
   213→			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
   214→		}
   215→		var fileFrom, fileTo int64
   216→		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
   217→			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
   218→		}
   219→		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
   220→			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
   221→		}
   222→		cf, err := readBinaryLevel(br)
   223→		if err != nil {
   224→			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
   225→		}
   226→		cf.From = fileFrom
   227→		cf.To = fileTo
   228→		return cf, nil
   229→
   230→	default:
   231→		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
   232→	}
   233→}
   234→
   235→// loadCheckpointFiles reads checkpoint files for a host directory and returns
   236→// the loaded CheckpointFiles and their filenames. Processes one file at a time
   237→// to avoid holding all checkpoint data in memory simultaneously.
   238→func loadCheckpointFiles(dir string, from int64) ([]*CheckpointFile, []string, error) {
   239→	entries, err := os.ReadDir(dir)
   240→	if err != nil {
   241→		return nil, nil, err
   242→	}
   243→
   244→	files, err := findFiles(entries, from, false)
   245→	if err != nil {
   246→		return nil, nil, err
   247→	}
   248→
   249→	if len(files) == 0 {
   250→		return nil, nil, nil
   251→	}
   252→
   253→	var checkpoints []*CheckpointFile
   254→	var processedFiles []string
   255→
   256→	for _, checkpoint := range files {
   257→		filename := filepath.Join(dir, checkpoint)
   258→		cf, err := loadCheckpointFileFromDisk(filename)
   259→		if err != nil {
   260→			continue
   261→		}
   262→		checkpoints = append(checkpoints, cf)
   263→		processedFiles = append(processedFiles, checkpoint)
   264→	}
   265→
   266→	return checkpoints, processedFiles, nil
   267→}
   268→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"bufio"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"

	pq "github.com/parquet-go/parquet-go"
)

// ParquetMetricRow is the long-format schema for archived metric data.
// One row per (host, metric, scope, scope_id, timestamp) data point.
// Sorted by (cluster, hostname, metric, timestamp) for optimal compression.
type ParquetMetricRow struct {
	Cluster   string  `parquet:"cluster"`
	Hostname  string  `parquet:"hostname"`
	Metric    string  `parquet:"metric"`
	Scope     string  `parquet:"scope"`
	ScopeID   string  `parquet:"scope_id"`
	Timestamp int64   `parquet:"timestamp"`
	Frequency int64   `parquet:"frequency"`
	Value     float32 `parquet:"value"`
}

// parseScopeFromName infers scope and scope_id from a child level name.
// Examples: "socket0" → ("socket", "0"), "core12" → ("core", "12"),
// "a0" (accelerator) → ("accelerator", "0").
// If the name doesn't match known patterns, it's used as-is for scope with empty scope_id.
func parseScopeFromName(name string) (string, string) {
	prefixes := []struct {
		prefix string
		scope  string
	}{
		{"socket", "socket"},
		{"memoryDomain", "memoryDomain"},
		{"core", "core"},
		{"hwthread", "hwthread"},
		{"cpu", "hwthread"},
		{"accelerator", "accelerator"},
	}

	for _, p := range prefixes {
		if len(name) > len(p.prefix) && name[:len(p.prefix)] == p.prefix {
			id := name[len(p.prefix):]
			if len(id) > 0 && id[0] >= '0' && id[0] <= '9' {
				return p.scope, id
			}
		}
	}

	return name, ""
}

// parquetArchiveWriter supports incremental writes to a Parquet file.
// Uses streaming writes to avoid accumulating all rows in memory.
type parquetArchiveWriter struct {
	writer *pq.GenericWriter[ParquetMetricRow]
	bw     *bufio.Writer
	f      *os.File
	batch  []ParquetMetricRow // reusable batch buffer
	count  int
}

const parquetBatchSize = 1024

// newParquetArchiveWriter creates a streaming Parquet writer with Zstd compression.
func newParquetArchiveWriter(filename string) (*parquetArchiveWriter, error) {
	if err := os.MkdirAll(filepath.Dir(filename), CheckpointDirPerms); err != nil {
		return nil, fmt.Errorf("creating archive directory: %w", err)
	}

	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil {
		return nil, fmt.Errorf("creating parquet file: %w", err)
	}

	bw := bufio.NewWriterSize(f, 1<<20) // 1MB write buffer

	writer := pq.NewGenericWriter[ParquetMetricRow](bw,
		pq.Compression(&pq.Zstd),
	)

	return &parquetArchiveWriter{
		writer: writer,
		bw:     bw,
		f:      f,
		batch:  make([]ParquetMetricRow, 0, parquetBatchSize),
	}, nil
}

// WriteCheckpointFile streams a CheckpointFile tree directly to Parquet rows,
// writing metrics in sorted order without materializing all rows in memory.
// Produces one row group per call (typically one host's data).
func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, cluster, hostname, scope, scopeID string) error {
	w.writeLevel(cf, cluster, hostname, scope, scopeID)

	// Flush remaining batch
	if len(w.batch) > 0 {
		if _, err := w.writer.Write(w.batch); err != nil {
			return fmt.Errorf("writing parquet rows: %w", err)
		}
		w.count += len(w.batch)
		w.batch = w.batch[:0]
	}

	if err := w.writer.Flush(); err != nil {
		return fmt.Errorf("flushing parquet row group: %w", err)
	}

	return nil
}

// writeLevel recursively writes metrics from a CheckpointFile level.
// Metric names and child names are sorted for deterministic, compression-friendly output.
func (w *parquetArchiveWriter) writeLevel(cf *CheckpointFile, cluster, hostname, scope, scopeID string) {
	// Sort metric names for deterministic order
	metricNames := make([]string, 0, len(cf.Metrics))
	for name := range cf.Metrics {
		metricNames = append(metricNames, name)
	}
	sort.Strings(metricNames)

	for _, metricName := range metricNames {
		cm := cf.Metrics[metricName]
		ts := cm.Start
		for _, v := range cm.Data {
			if !v.IsNaN() {
				w.batch = append(w.batch, ParquetMetricRow{
					Cluster:   cluster,
					Hostname:  hostname,
					Metric:    metricName,
					Scope:     scope,
					ScopeID:   scopeID,
					Timestamp: ts,
					Frequency: cm.Frequency,
					Value:     float32(v),
				})

				if len(w.batch) >= parquetBatchSize {
					w.writer.Write(w.batch)
					w.count += len(w.batch)
					w.batch = w.batch[:0]
				}
			}
			ts += cm.Frequency
		}
	}

	// Sort child names for deterministic order
	childNames := make([]string, 0, len(cf.Children))
	for name := range cf.Children {
		childNames = append(childNames, name)
	}
	sort.Strings(childNames)

	for _, childName := range childNames {
		childScope, childScopeID := parseScopeFromName(childName)
		w.writeLevel(cf.Children[childName], cluster, hostname, childScope, childScopeID)
	}
}

// Close finalises the Parquet file (footer, buffered I/O, file handle).
func (w *parquetArchiveWriter) Close() error {
	if err := w.writer.Close(); err != nil {
		w.f.Close()
		return fmt.Errorf("closing parquet writer: %w", err)
	}

	if err := w.bw.Flush(); err != nil {
		w.f.Close()
		return fmt.Errorf("flushing parquet file: %w", err)
	}

	return w.f.Close()
}

// loadCheckpointFileFromDisk reads a JSON or binary checkpoint file and returns
// a CheckpointFile. Used by the Parquet archiver to read checkpoint data
// before converting it to Parquet format.
func loadCheckpointFileFromDisk(filename string) (*CheckpointFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	ext := filepath.Ext(filename)
	switch ext {
	case ".json":
		cf := &CheckpointFile{}
		br := bufio.NewReader(f)
		if err := json.NewDecoder(br).Decode(cf); err != nil {
			return nil, fmt.Errorf("decoding JSON checkpoint %s: %w", filename, err)
		}
		return cf, nil

	case ".bin":
		br := bufio.NewReader(f)
		var magic uint32
		if err := binary.Read(br, binary.LittleEndian, &magic); err != nil {
			return nil, fmt.Errorf("reading magic from %s: %w", filename, err)
		}
		if magic != snapFileMagic {
			return nil, fmt.Errorf("invalid snapshot magic in %s: 0x%08X", filename, magic)
		}
		var fileFrom, fileTo int64
		if err := binary.Read(br, binary.LittleEndian, &fileFrom); err != nil {
			return nil, fmt.Errorf("reading from-timestamp from %s: %w", filename, err)
		}
		if err := binary.Read(br, binary.LittleEndian, &fileTo); err != nil {
			return nil, fmt.Errorf("reading to-timestamp from %s: %w", filename, err)
		}
		cf, err := readBinaryLevel(br)
		if err != nil {
			return nil, fmt.Errorf("reading binary level from %s: %w", filename, err)
		}
		cf.From = fileFrom
		cf.To = fileTo
		return cf, nil

	default:
		return nil, fmt.Errorf("unsupported checkpoint extension: %s", ext)
	}
}

// loadCheckpointFiles reads checkpoint files for a host directory and returns
// the loaded CheckpointFiles and their filenames. Processes one file at a time
// to avoid holding all checkpoint data in memory simultaneously.
func loadCheckpointFiles(dir string, from int64) ([]*CheckpointFile, []string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, nil, err
	}

	files, err := findFiles(entries, from, false)
	if err != nil {
		return nil, nil, err
	}

	if len(files) == 0 {
		return nil, nil, nil
	}

	var checkpoints []*CheckpointFile
	var processedFiles []string

	for _, checkpoint := range files {
		filename := filepath.Join(dir, checkpoint)
		cf, err := loadCheckpointFileFromDisk(filename)
		if err != nil {
			continue
		}
		checkpoints = append(checkpoints, cf)
		processedFiles = append(processedFiles, checkpoint)
	}

	return checkpoints, processedFiles, nil
}


Now I have a clear understanding of the issue. Let me launch a Plan agent to design the fix.


Agent

Plan parquet row group fix

Prompt:
Design a fix for a parquet row group limit issue in the metric store archiver.

## Problem

The error is: "the limit of 32767 row groups has been reached"

This happens in `pkg/metricstore/archive.go:archiveCheckpoints()` which creates ONE parquet file per cluster. For each host's checkpoint file, `WriteCheckpointFile()` in `pkg/metricstore/parquetArchive.go` calls `writer.Flush()`, creating one row group per checkpoint file.

For large clusters like "fritz" with many hosts (e.g., 1000+) and multiple checkpoint files per host, total flushes easily exceed 32767.

## Current flow (archive.go lines 268-280):

```go
for r := range results {
    for _, cf := range r.checkpoints {
        // Each call creates a row group via Flush()
        writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", "")
    }
}
```

## WriteCheckpointFile (parquetArchive.go lines 103-120):

```go
func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, ...) error {
    w.writeLevel(cf, cluster, hostname, scope, scopeID)
    // Flush remaining batch
    if len(w.batch) > 0 {
        w.writer.Write(w.batch)
        w.count += len(w.batch)
        w.batch = w.batch[:0]
    }
    // THIS creates a new row group every time
    if err := w.writer.Flush(); err != nil {
        return fmt.Errorf("flushing parquet row group: %w", err)
    }
    return nil
}
```

## Design constraints

- The parquet-go library hard-limits row groups to 32767 per file
- We want good compression (data sorted by cluster, hostname, metric, timestamp)
- Don't accumulate all rows in memory — streaming is important
- Performance: this is HPC monitoring at scale

## Possible approaches

1. **Remove per-checkpoint Flush, flush periodically instead**: Instead of flushing after every checkpoint file, flush after accumulating a certain number of rows (e.g., every 100k rows or every N checkpoints). This reduces row groups dramatically while still bounding memory.

2. **Split into multiple parquet files**: When approaching the row group limit, close the current file and open a new one.

3. **Flush per host instead of per checkpoint**: Since multiple checkpoints can belong to the same host, only flush once per host rather than per checkpoint file.

Which approach is best? Consider that:
- A host may have 1 to ~hundreds of checkpoint files
- A cluster may have ~100 to ~3000+ hosts
- Flushing per host would give at most ~3000 row groups (within limit for most clusters)
- But combining approaches 1+2 would be most robust

Please design the recommended implementation approach with specific code changes needed.


Design a fix for a parquet row group limit issue in the metric store archiver.

## Problem

The error is: "the limit of 32767 row groups has been reached"

This happens in `pkg/metricstore/archive.go:archiveCheckpoints()` which creates ONE parquet file per cluster. For each host's checkpoint file, `WriteCheckpointFile()` in `pkg/metricstore/parquetArchive.go` calls `writer.Flush()`, creating one row group per checkpoint file.

For large clusters like "fritz" with many hosts (e.g., 1000+) and multiple checkpoint files per host, total flushes easily exceed 32767.

## Current flow (archive.go lines 268-280):

```go
for r := range results {
    for _, cf := range r.checkpoints {
        // Each call creates a row group via Flush()
        writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", "")
    }
}
```

## WriteCheckpointFile (parquetArchive.go lines 103-120):

```go
func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, ...) error {
    w.writeLevel(cf, cluster, hostname, scope, scopeID)
    // Flush remaining batch
    if len(w.batch) > 0 {
        w.writer.Write(w.batch)
        w.count += len(w.batch)
        w.batch = w.batch[:0]
    }
    // THIS creates a new row group every time
    if err := w.writer.Flush(); err != nil {
        return fmt.Errorf("flushing parquet row group: %w", err)
    }
    return nil
}
```

## Design constraints

- The parquet-go library hard-limits row groups to 32767 per file
- We want good compression (data sorted by cluster, hostname, metric, timestamp)
- Don't accumulate all rows in memory — streaming is important
- Performance: this is HPC monitoring at scale

## Possible approaches

1. **Remove per-checkpoint Flush, flush periodically instead**: Instead of flushing after every checkpoint file, flush after accumulating a certain number of rows (e.g., every 100k rows or every N checkpoints). This reduces row groups dramatically while still bounding memory.

2. **Split into multiple parquet files**: When approaching the row group limit, close the current file and open a new one.

3. **Flush per host instead of per checkpoint**: Since multiple checkpoints can belong to the same host, only flush once per host rather than per checkpoint file.

Which approach is best? Consider that:
- A host may have 1 to ~hundreds of checkpoint files
- A cluster may have ~100 to ~3000+ hosts
- Flushing per host would give at most ~3000 row groups (within limit for most clusters)
- But combining approaches 1+2 would be most robust

Please design the recommended implementation approach with specific code changes needed.

Prompt:
Design a fix for a parquet row group limit issue in the metric store archiver.

## Problem

The error is: "the limit of 32767 row groups has been reached"

This happens in `pkg/metricstore/archive.go:archiveCheckpoints()` which creates ONE parquet file per cluster. For each host's checkpoint file, `WriteCheckpointFile()` in `pkg/metricstore/parquetArchive.go` calls `writer.Flush()`, creating one row group per checkpoint file.

For large clusters like "fritz" with many hosts (e.g., 1000+) and multiple checkpoint files per host, total flushes easily exceed 32767.

## Current flow (archive.go lines 268-280):

```go
for r := range results {
    for _, cf := range r.checkpoints {
        // Each call creates a row group via Flush()
        writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", "")
    }
}
```

## WriteCheckpointFile (parquetArchive.go lines 103-120):

```go
func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, ...) error {
    w.writeLevel(cf, cluster, hostname, scope, scopeID)
    // Flush remaining batch
    if len(w.batch) > 0 {
        w.writer.Write(w.batch)
        w.count += len(w.batch)
        w.batch = w.batch[:0]
    }
    // THIS creates a new row group every time
    if err := w.writer.Flush(); err != nil {
        return fmt.Errorf("flushing parquet row group: %w", err)
    }
    return nil
}
```

## Design constraints

- The parquet-go library hard-limits row groups to 32767 per file
- We want good compression (data sorted by cluster, hostname, metric, timestamp)
- Don't accumulate all rows in memory — streaming is important
- Performance: this is HPC monitoring at scale

## Possible approaches

1. **Remove per-checkpoint Flush, flush periodically instead**: Instead of flushing after every checkpoint file, flush after accumulating a certain number of rows (e.g., every 100k rows or every N checkpoints). This reduces row groups dramatically while still bounding memory.

2. **Split into multiple parquet files**: When approaching the row group limit, close the current file and open a new one.

3. **Flush per host instead of per checkpoint**: Since multiple checkpoints can belong to the same host, only flush once per host rather than per checkpoint file.

Which approach is best? Consider that:
- A host may have 1 to ~hundreds of checkpoint files
- A cluster may have ~100 to ~3000+ hosts
- Flushing per host would give at most ~3000 row groups (within limit for most clusters)
- But combining approaches 1+2 would be most robust

Please design the recommended implementation approach with specific code changes needed.


Bash

Show staged changes to archive.go


pkg/metricstore/archive.go
  @@ -22,6 +22,7 @@
  +		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
   		// Run as Archiver
   		cleanUpWorker(wg, ctx,
   			Keys.RetentionInMemory,
  @@ -43,7 +44,6 @@
  -
   		d, err := time.ParseDuration(interval)
   		if err != nil {
   			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
  @@ -99,8 +99,8 @@
  -		dir            string
  -		cluster, host  string
  +		dir           string
  +		cluster, host string
   	}

   	var wg sync.WaitGroup
  +3 -3


No files found


find /Users/jan/prg/ClusterCockpit/cc-backend/vendor -path "*/parquet-go" -type d 2>/dev/null | head -5

Find parquet-go vendor directory


(Bash completed with no output)


ls /Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go*/writer*.go 2>/dev/null || find /Users/jan/go/pkg/mod -path "*/parquet-go*/writer.go" 2>/dev/null | head -5

Find parquet-go writer source


Find parquet-go dependency version


	github.com/parquet-go/parquet-go v0.29.0
	github.com/parquet-go/bitpack v1.0.0 // indirect
	github.com/parquet-go/jsonlite v1.4.0 // indirect


271-	return w.base.WriteRows(rows)
272-}
273-
274:func (w *GenericWriter[T]) WriteRowGroup(rowGroup RowGroup) (int64, error) {
275:	return w.base.WriteRowGroup(rowGroup)
276-}
277-
278-// SetKeyValueMetadata sets a key/value pair in the Parquet file metadata.
--
312-	return w.base.File()
313-}
314-
315:// ConcurrentRowGroupWriter is a row group writer that can be used to write row groups
316:// in parallel. Multiple row groups can be created concurrently and written to independently,
317:// but they must be committed serially to maintain the order of row groups in the file.
318-//
319-// See BeginRowGroup for more information on how this can be used.
320-//
321:// While multiple row groups can be created concurrently, a single row group must be written
322-// sequentially.
323-type ConcurrentRowGroupWriter struct {
324-	writer      *writer
--
333-}
334-
335-// BeginRowGroup returns a new ConcurrentRowGroupWriter that can be written to in parallel with
336:// other row groups. However these need to be committed back to the writer serially using the
337:// Commit method on the row group.
338-//
339-/ Example usage could look something like:
340-//
--
401-// The Writer type optimizes for minimal memory usage, each page is written as
402-// soon as it has been filled so only a single page per column needs to be held
403-// in memory and as a result, there are no opportunities to sort rows within an
404:// entire row group. Programs that need to produce parquet files with sorted
405:// row groups should use the Buffer type to buffer and sort the rows prior to
406-// writing them to a Writer.
407-//
408-// For programs building with Go 1.18 or later, the GenericWriter[T] type
--
468-	return nil
469-}
470-
471:// Flush flushes all buffers into a row group to the underlying io.Writer.
472-//
473-// Flush is called automatically on Close, it is only useful to call explicitly
474:// if the application needs to limit the size of row groups or wants to produce
475:// multiple row groups per file.
476-//
477:// If the writer attempts to create more than MaxRowGroups row groups the method
478-// returns ErrTooManyRowGroups.
479-func (w *Writer) Flush() error {
480-	if w.writer != nil {
--
526-	return w.writer.WriteRows(rows)
527-}
528-
529:// WriteRowGroup writes a row group to the parquet file.
530-//
531-// Buffered rows will be flushed prior to writing rows from the group, unless
532:// the row group was empty in which case nothing is written to the file.
533-//
534:// The content of the row group is flushed to the writer; after the method
535:// returns successfully, the row group will be empty and in ready to be reused.
536:func (w *Writer) WriteRowGroup(rowGroup RowGroup) (int64, error) {
537:	rowGroupSchema := rowGroup.Schema()
538-	switch {
539:	case rowGroupSchema == nil:
540-		return 0, ErrRowGroupSchemaMissing
541-	case w.schema == nil:
542:		w.configure(rowGroupSchema)
543:	case !EqualNodes(w.schema, rowGroupSchema):
544-		return 0, ErrRowGroupSchemaMismatch
545-	}
546-	if err := w.writer.flush(); err != nil {
547-		return 0, err
548-	}
549:	w.writer.currentRowGroup.configureBloomFilters(rowGroup.ColumnChunks())
550:	rows := rowGroup.Rows()
551-	defer rows.Close()
552-	n, err := CopyRows(w.writer, rows)
553-	if err != nil {
554-		return n, err
555-	}
556:	return w.writer.writeRowGroup(w.writer.currentRowGroup, rowGroup.Schema(), rowGroup.SortingColumns())
557-}
558-
559-// ReadRowsFrom reads rows from the reader passed as arguments and writes them
--
581-func (w *Writer) Schema() *Schema { return w.schema }
582-
583-// Size returns an estimate of the current file size in bytes, including all
584:// completed row groups and the current in-progress row group.
585-//
586-// The estimate includes bytes already written to the underlying writer plus
587:// the estimated size of buffered data in the current row group. The buffered
588-// data has not been compressed yet, so this is an upper-bound estimate.
589-//
590-// The size of the footer metadata is not included because it is variable and
--
628-func (w *Writer) ColumnWriters() []*ColumnWriter { return w.writer.currentRowGroup.columns }
629-
630-// BeginRowGroup returns a new ConcurrentRowGroupWriter that can be written to in parallel with
631:// other row groups. However these need to be committed back to the writer serially using the
632:// Commit method on the row group.
633-func (w *Writer) BeginRowGroup() *ConcurrentRowGroupWriter {
634-	return newConcurrentRowGroupWriter(w.writer, w.config)
635-}
--
839-	}
840-}
841-
842:// Schema returns the schema for this row group.
843-func (rg *ConcurrentRowGroupWriter) Schema() *Schema {
844-	return rg.config.Schema
845-}
846-
847:// ColumnWriters returns the column writers for this row group, allowing
848-// direct access to write values to individual columns.
849-func (rg *ConcurrentRowGroupWriter) ColumnWriters() []*ColumnWriter {
850-	return rg.columns
851-}
852-
853:// Flush flushes any buffered data in the row group's column writers.
854-// This could be called before Commit to ensure all data pages are flushed.
855-func (rg *ConcurrentRowGroupWriter) Flush() error {
856-	for _, c := range rg.columns {
--
861-	return nil
862-}
863-
864:// Size returns an estimate of the current row group size in bytes.
865-//
866-// The estimate sums encoded pages, buffered values not yet encoded, dictionary
867-// pages, and bloom filters for each column. Because buffered values have not
--
881-	return size
882-}
883-
884:// Commit commits the row group to the parent writer, returning the number
885-// of rows written and an error if any. This method must be called serially
886:// (not concurrently) to maintain row group order in the file.
887-//
888-// If the parent writer has any pending rows buffered, they will be flushed
889:// before this row group is written.
890-//
891:// After Commit returns successfully, the row group will be empty and can
892-// be reused.
893-func (rg *ConcurrentRowGroupWriter) Commit() (int64, error) {
894-	if err := rg.writer.flush(); err != nil {
--
897-	return rg.writer.writeRowGroup(rg, nil, nil)
898-}
899-
900:// WriteRows writes rows to the row group.
901-func (rg *ConcurrentRowGroupWriter) WriteRows(rows []Row) (int, error) {
902-	return rg.writeRows(len(rows), func(start, end int) (int, error) {
903-		defer func() {
--
974-
975-	columnOrders   []format.ColumnOrder
976-	schemaElements []format.SchemaElement
977:	rowGroups      []format.RowGroup
978-	columnIndexes  [][]format.ColumnIndex
979-	offsetIndexes  [][]format.OffsetIndex
980-	sortingColumns []format.SortingColumn
--
1074-		w.writer.Reset(w.buffer)
1075-	}
1076-	w.currentRowGroup.reset()
1077:	for i := range w.rowGroups {
1078:		w.rowGroups[i].Reset()
1079-	}
1080-	for i := range w.columnIndexes {
1081-		for j := range w.columnIndexes[i] {
--
1089-		}
1090-		w.offsetIndexes[i] = w.offsetIndexes[i][:0]
1091-	}
1092:	w.rowGroups = w.rowGroups[:0]
1093-	w.columnIndexes = w.columnIndexes[:0]
1094-	w.offsetIndexes = w.offsetIndexes[:0]
1095-	w.fileMetaData = format.FileMetaData{}
--
1129-
1130-func (w *writer) writeFileFooter() error {
1131-	/ The page index is composed of two sections: column and offset indexes.
1132:	// They are written after the row groups, right before the footer (which
1133-	// is written by the parent Writer.Close call).
1134-	//
1135-	// This section both writes the page index and generates the values of
--
1144-	encoder := thrift.NewEncoder(protocol.NewWriter(&w.writer))
1145-
1146-	for i, columnIndexes := range w.columnIndexes {
1147:		rowGroup := &w.rowGroups[i]
1148-		for j := range columnIndexes {
1149:			column := &rowGroup.Columns[j]
1150-			column.ColumnIndexOffset = w.writer.offset
1151-			if err := encoder.Encode(&columnIndexes[j]); err != nil {
1152-				return err
--
1156-	}
1157-
1158-	for i, offsetIndexes := range w.offsetIndexes {
1159:		rowGroup := &w.rowGroups[i]
1160-		for j := range offsetIndexes {
1161:			column := &rowGroup.Columns[j]
1162-			column.OffsetIndexOffset = w.writer.offset
1163-			if err := encoder.Encode(&offsetIndexes[j]); err != nil {
1164-				return err
--
1168-	}
1169-
1170-	numRows := int64(0)
1171:	for rowGroupIndex := range w.rowGroups {
1172:		numRows += w.rowGroups[rowGroupIndex].NumRows
1173-	}
1174-
1175-	// We implemented the parquet specification version 2+, which is represented
--
1183-		Version:          parquetFileFormatVersion,
1184-		Schema:           w.schemaElements,
1185-		NumRows:          numRows,
1186:		RowGroups:        w.rowGroups,
1187-		KeyValueMetadata: w.metadata,
1188-		CreatedBy:        w.createdBy,
1189-		ColumnOrders:     w.columnOrders,
--
1201-	return err
1202-}
1203-
1204:func (w *writer) writeRowGroup(rg *ConcurrentRowGroupWriter, rowGroupSchema *Schema, rowGroupSortingColumns []SortingColumn) (int64, error) {
1205-	if len(rg.columns) == 0 {
1206-		return 0, nil
1207-	}
--
1210-		return 0, nil
1211-	}
1212-
1213:	rowGroupIndex := len(w.rowGroups)
1214-	columnIndexIndex := len(w.columnIndexes)
1215-	offsetIndexIndex := len(w.offsetIndexes)
1216-
1217:	if len(w.rowGroups) == MaxRowGroups {
1218-		return 0, ErrTooManyRowGroups
1219-	}
1220-
--
1250-		if c.dictionary != nil {
1251-			c.columnChunk.MetaData.DictionaryPageOffset = w.writer.offset
1252-			if err := c.writeDictionaryPage(&w.writer, c.dictionary); err != nil {
1253:				return 0, fmt.Errorf("writing dictionary page of row group colum %d: %w", i, err)
1254-			}
1255-		}
1256-
--
1271-			return 0, fmt.Errorf("resetting parquet page buffer to the start expected offset zero but got %d", offset)
1272-		}
1273-		if _, err := io.Copy(&w.writer, c.pageBuffer); err != nil {
1274:			return 0, fmt.Errorf("writing buffered pages of row group column %d: %w", i, err)
1275-		}
1276-	}
1277-
--
1298-	}
1299-
1300-	var reuseRowGroup *format.RowGroup = nil
1301:	if cap(w.rowGroups) > rowGroupIndex {
1302:		w.rowGroups = w.rowGroups[:rowGroupIndex+1] // Extend the slice by one element
1303:		reuseRowGroup = &w.rowGroups[rowGroupIndex] // Pointer to the last element
1304-	}
1305-
1306-	sortingColumns := w.sortingColumns
1307:	if len(sortingColumns) == 0 && len(rowGroupSortingColumns) > 0 {
1308:		scLen := len(rowGroupSortingColumns)
1309-		if reuseRowGroup == nil {
1310-			sortingColumns = make([]format.SortingColumn, 0, scLen)
1311-		} else {
1312-			sortingColumns = reuseRowGroup.SortingColumns
1313-		}
1314:		forEachLeafColumnOf(rowGroupSchema, func(leaf leafColumn) {
1315:			if sortingIndex := searchSortingColumn(rowGroupSortingColumns, leaf.path); sortingIndex < len(sortingColumns) {
1316-				sortingColumns[sortingIndex] = format.SortingColumn{
1317-					ColumnIdx:  int32(leaf.columnIndex),
1318:					Descending: rowGroupSortingColumns[sortingIndex].Descending(),
1319:					NullsFirst: rowGroupSortingColumns[sortingIndex].NullsFirst(),
1320-				}
1321-			}
1322-		})
--
1353-		w.offsetIndexes = append(w.offsetIndexes, offsetIndex)
1354-	}
1355-
1356:	// If the existing row group slice is full, we need to create a new element
1357-	var columns []format.ColumnChunk = nil
1358-	if reuseRowGroup == nil {
1359-		columns = slices.Clone(rg.columnChunk)
--
1371-	}
1372-
1373-	if reuseRowGroup == nil {
1374:		w.rowGroups = append(w.rowGroups, format.RowGroup{
1375-			Columns:             columns,
1376-			TotalByteSize:       totalByteSize,
1377-			NumRows:             numRows,
1378-			SortingColumns:      sortingColumns,
1379-			FileOffset:          fileOffset,
1380-			TotalCompressedSize: totalCompressedSize,
1381:			Ordinal:             int16(len(w.rowGroups)),
1382-		})
1383-	} else {
1384-		reuseRowGroup.Columns = columns
--
1387-		reuseRowGroup.SortingColumns = sortingColumns
1388-		reuseRowGroup.FileOffset = fileOffset
1389-		reuseRowGroup.TotalCompressedSize = totalCompressedSize
1390:		reuseRowGroup.Ordinal = int16(len(w.rowGroups) - 1)
1391-	}
1392-
1393-	return numRows, nil
--
1528-	columnIndex          ColumnIndexer
1529-	columnBuffer         ColumnBuffer
1530-	plainColumnBuffer    ColumnBuffer // Retained plain buffer for fallback after lazy creation
1531:	originalColumnBuffer ColumnBuffer // Original buffer to restore after row group flush
1532-	columnFilter         BloomFilterColumn
1533-	encoding             encoding.Encoding
1534-	originalEncoding     encoding.Encoding // Original encoding before any changes
--
1591-		c.pageBuffer = nil
1592-	}
1593-	c.numPages = 0
1594:	// Bloom filters may change in size between row groups, but we retain the
1595-	// buffer to avoid reallocating large memory blocks.
1596-	c.filter = c.filter[:0]
1597-	c.numRows = 0
1598:	// Reset the fields of column chunks that change between row groups,
1599-	// but keep the ones that remain unchanged.
1600-	c.columnChunk.MetaData.NumValues = 0
1601-	c.columnChunk.MetaData.TotalUncompressedSize = 0
--
1808-func (c *ColumnWriter) WriteRowValues(rows []Value) (int, error) {
1809-	var startingRows int64
1810-	if c.columnBuffer == nil {
1811:		// Lazily create the row group column so we don't need to allocate it if
1812-		// rows are not written individually to the column.
1813-		c.columnBuffer = c.newColumnBuffer()
1814-		c.originalColumnBuffer = c.columnBuffer
--
1841-func (c *ColumnWriter) writeValues(values []Value) (numValues int, err error) {
1842-	if c.columnBuffer == nil {
1843-		c.columnBuffer = c.newColumnBuffer()
1844:		// Save the original dictionary-encoding buffer to restore after row group flush
1845-		if c.originalColumnBuffer == nil {
1846-			c.originalColumnBuffer = c.columnBuffer
1847-		}
--
1896-
1897-	if page.Dictionary() == nil && len(c.filter) > 0 {
1898-		// When the writer knows the number of values in advance (e.g. when
1899:		// writing a full row group), the filter encoding is set and the page
1900-		// can be directly applied to the filter, which minimizes memory usage
1901-		// since there is no need to buffer the values in order to determine
1902-		// the size of the filter.
--
2080-	/ Note: We are NOT re-encoding existing pages. This means:
2081-	// - Pages written before the limit was hit will remain dictionary-encoded
2082-	// - Pages written after will be PLAIN-encoded
2083:	// - The dictionary page will still be written at row group flush
2084-	// - Mixed encodings in the same column chunk are valid per Parquet spec
2085-	c.hasSwitchedToPlain = true
2086-	return nil


/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/errors.go:54:	// ErrTooManyRowGroups is returned when attempting to generate a parquet
/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/errors.go:55:	// file with more than MaxRowGroups row groups.
/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/errors.go:56:	ErrTooManyRowGroups = errors.New("the limit of 32767 row groups has been reached")
/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/writer.go:260:		if err != ErrTooManyRowGroups {
/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/writer.go:477:// If the writer attempts to create more than MaxRowGroups row groups the method
/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/writer.go:478:// returns ErrTooManyRowGroups.
/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/writer.go:940:			return written, ErrTooManyRowGroups
/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/writer.go:1217:	if len(w.rowGroups) == MaxRowGroups {
/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/writer.go:1218:		return 0, ErrTooManyRowGroups
/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/writer.go:1402:		if err != ErrTooManyRowGroups {
/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/limits.go:23:	// MaxRowGroups is the maximum number of row groups which can be contained
/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/limits.go:29:	MaxRowGroups = math.MaxInt16


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-134-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-135-
pkg/metricstore/metricstore.go:136:	// Set NumWorkers from config or use default
pkg/metricstore/metricstore.go:137:	if Keys.NumWorkers <= 0 {
pkg/metricstore/metricstore.go:138:		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-139-	}
pkg/metricstore/metricstore.go:140:	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-141-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-142-	// Pass the config.MetricStoreKeys
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-186-	if Keys.Subscriptions != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-187-		wg.Go(func() {
pkg/metricstore/metricstore.go:188:			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-189-				cclog.Fatal(err)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go-190-			}
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-106-	var wg sync.WaitGroup
pkg/metricstore/archive.go-107-	n, errs := int32(0), int32(0)
pkg/metricstore/archive.go:108:	work := make(chan workItem, Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-109-
pkg/metricstore/archive.go:110:	wg.Add(Keys.NumWorkers)
pkg/metricstore/archive.go:111:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-112-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-113-			defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-202-		}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-203-
pkg/metricstore/archive.go:204:		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
pkg/metricstore/archive.go-205-		results := make(chan hostResult, 2)
pkg/metricstore/archive.go-206-		work := make(chan struct {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-207-			dir, host string
pkg/metricstore/archive.go:208:		}, Keys.NumWorkers)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-209-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-210-		var wg sync.WaitGroup
pkg/metricstore/archive.go-211-		errs := int32(0)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-212-
pkg/metricstore/archive.go:213:		wg.Add(Keys.NumWorkers)
pkg/metricstore/archive.go:214:		for w := 0; w < Keys.NumWorkers; w++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-215-			go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go-216-				defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-11-//
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-12-//	MetricStoreConfig (Keys)
pkg/metricstore/config.go:13://	├─ NumWorkers: Parallel checkpoint/archive workers
pkg/metricstore/config.go-14-/	├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval)
pkg/metricstore/config.go-15-/	├─ MemoryCap: Memory limit in bytes (triggers forceFree)
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-112-//
pkg/metricstore/config.go-113-/ Fields:
pkg/metricstore/config.go:114://   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
pkg/metricstore/config.go-115-/   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
pkg/metricstore/config.go-116-/   - MemoryCap:         Max memory in GB for buffer data (0 = unlimited); triggers forceFree when exceeded
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-122-	// Number of concurrent workers for checkpoint and archive operations.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go-123-	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
pkg/metricstore/config.go:124:	NumWorkers         int            `json:"num-workers"`
pkg/metricstore/config.go-125-	RetentionInMemory  string         `json:"retention-in-memory"`
pkg/metricstore/config.go-126-	CheckpointInterval string         `json:"checkpoint-interval,omitempty"`
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-182-//
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-183-// Metrics at root and cluster levels are skipped. One file per host is created.
pkg/metricstore/checkpoint.go:184:// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-185-// at a time, allowing concurrent writes/reads to other hosts.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-186-//
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-220-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-221-	var wg sync.WaitGroup
pkg/metricstore/checkpoint.go:222:	wg.Add(Keys.NumWorkers)
pkg/metricstore/checkpoint.go:223:	work := make(chan workItem, Keys.NumWorkers*2)
pkg/metricstore/checkpoint.go:224:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-225-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-226-			defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-395-func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-396-	var wg sync.WaitGroup
pkg/metricstore/checkpoint.go:397:	work := make(chan [2]string, Keys.NumWorkers*4)
pkg/metricstore/checkpoint.go-398-	n, errs := int32(0), int32(0)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-399-
pkg/metricstore/checkpoint.go:400:	wg.Add(Keys.NumWorkers)
pkg/metricstore/checkpoint.go:401:	for worker := 0; worker < Keys.NumWorkers; worker++ {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-402-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go-403-			defer wg.Done()
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-150-	}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-151-
pkg/metricstore/walCheckpoint.go:152:	walNumShards = max(Keys.NumWorkers, 1)
pkg/metricstore/walCheckpoint.go-153-	chBufSize := max(65536/walNumShards, 1024)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-154-
--
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-703-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-704-	var wg sync.WaitGroup
pkg/metricstore/walCheckpoint.go:705:	wg.Add(Keys.NumWorkers)
pkg/metricstore/walCheckpoint.go:706:	work := make(chan workItem, Keys.NumWorkers*2)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-707-
pkg/metricstore/walCheckpoint.go:708:	for range Keys.NumWorkers {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-709-		go func() {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go-710-			defer wg.Done()


No matches found


Good. Now I have full context. Here is the implementation plan.

---

## Implementation Plan: Fix Parquet Row Group Limit

### Problem Analysis

Every call to `WriteCheckpointFile()` ends with `w.writer.Flush()`, which creates one row group. The parquet-go library hard-limits this to `MaxRowGroups = 32767` (math.MaxInt16). For a cluster with 3000 hosts and even just 11 checkpoint files per host, that is 33000 flushes, exceeding the limit.

### Recommended Approach: Combined Strategy (Approaches 1 + 3)

The fix has two parts:

1. **Move the `Flush()` call out of `WriteCheckpointFile` and into the caller in `archive.go`**, flushing once per host (after all checkpoint files for that host are written). This is the primary fix -- it is the natural boundary since each `hostResult` groups all checkpoints for a single host.

2. **Add a safety counter to `parquetArchiveWriter`** that tracks flush count. If the counter approaches the limit (e.g., at 30000), return a specific sentinel error so the caller can handle it gracefully.

This gives at most ~3000 row groups for a 3000-node cluster, well within the 32767 limit. The safety counter protects against future scenarios (e.g., if clusters grow to 30000+ nodes, or if the code is called differently).

### Detailed Changes

#### File 1: `pkg/metricstore/parquetArchive.go`

**Change A: Remove `Flush()` from `WriteCheckpointFile`, add separate `FlushRowGroup` method**

In `WriteCheckpointFile` (lines 103-120):
- Remove the `w.writer.Flush()` call (lines 115-117)
- Keep only the batch-draining logic (writing remaining rows in `w.batch`)

Add a new public method:

```go
// FlushRowGroup finalizes the current buffered rows as a row group.
// Should be called after writing all checkpoint files for one logical unit (e.g., one host).
func (w *parquetArchiveWriter) FlushRowGroup() error {
    w.rowGroupCount++
    if err := w.writer.Flush(); err != nil {
        return fmt.Errorf("flushing parquet row group: %w", err)
    }
    return nil
}
```

**Change B: Add `rowGroupCount` field to `parquetArchiveWriter` struct**

Add a `rowGroupCount int` field to the struct (line 65-71). This is informational -- the parquet-go library already enforces the hard limit, but having this counter lets us log useful diagnostics.

#### File 2: `pkg/metricstore/archive.go`

**Change: Flush once per host instead of per checkpoint file**

In `archiveCheckpoints()`, the loop at lines 268-281 currently calls `WriteCheckpointFile` for each checkpoint (which flushes each time). Change to:

```go
for r := range results {
    if writeErr == nil {
        for _, cf := range r.checkpoints {
            if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
                writeErr = err
                break
            }
        }
        // Flush one row group per host (all checkpoints combined)
        if writeErr == nil {
            if err := writer.FlushRowGroup(); err != nil {
                writeErr = err
            }
        }
    }
    toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
}
```

This ensures exactly one row group per host per cluster, giving at most ~3000 row groups for a 3000-node cluster.

### Why Not the Other Approaches

- **Approach 2 (split files)**: Adds significant complexity -- tracking multiple files, naming conventions, readers needing to handle multi-file archives. Unnecessary since per-host flushing keeps us well under the limit.

- **Periodic flush by row count**: Would break the natural host-boundary grouping. A row group per host is semantically clean and gives excellent compression (all data for one host is contiguous and sorted by metric+timestamp).

### Compression Benefit

Flushing per host instead of per checkpoint is actually better for compression. Multiple checkpoint files for the same host now end up in a single row group, keeping hostname, cluster, and metric columns highly repetitive within each group. The data is already sorted by (cluster, hostname, metric, timestamp) due to `writeLevel`'s sorted iteration.

### Edge Case: Very Large Clusters (30000+ nodes)

If this ever becomes a concern, the fix would be to add file splitting at that point. But since even the largest known HPC clusters (Frontier at ~9400 nodes) are well under 32767, this is not needed now. The `rowGroupCount` field makes it easy to add later.

### Testing

- Verify the existing test patterns for the parquet archiver. The key assertion is that `WriteCheckpointFile` no longer calls `Flush()` and `FlushRowGroup()` is called from the archive loop.
- For a cluster with N hosts, the parquet file should contain exactly N row groups.

### Critical Files for Implementation

- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/parquetArchive.go` - Remove Flush from WriteCheckpointFile, add FlushRowGroup method, add rowGroupCount field
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go` - Call FlushRowGroup once per host in the archiveCheckpoints loop
- `/Users/jan/go/pkg/mod/github.com/parquet-go/parquet-go@v0.29.0/writer.go` - Reference: understand Flush/row group semantics (read-only, library code)

agentId: a74b5803a0410fec5 (use SendMessage with to: 'a74b5803a0410fec5' to continue this agent)
<usage>total_tokens: 33433
tool_uses: 12
duration_ms: 140246</usage>

Prompt:
Design a fix for a parquet row group limit issue in the metric store archiver.

## Problem

The error is: "the limit of 32767 row groups has been reached"

This happens in `pkg/metricstore/archive.go:archiveCheckpoints()` which creates ONE parquet file per cluster. For each host's checkpoint file, `WriteCheckpointFile()` in `pkg/metricstore/parquetArchive.go` calls `writer.Flush()`, creating one row group per checkpoint file.

For large clusters like "fritz" with many hosts (e.g., 1000+) and multiple checkpoint files per host, total flushes easily exceed 32767.

## Current flow (archive.go lines 268-280):

```go
for r := range results {
    for _, cf := range r.checkpoints {
        // Each call creates a row group via Flush()
        writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", "")
    }
}
```

## WriteCheckpointFile (parquetArchive.go lines 103-120):

```go
func (w *parquetArchiveWriter) WriteCheckpointFile(cf *CheckpointFile, ...) error {
    w.writeLevel(cf, cluster, hostname, scope, scopeID)
    // Flush remaining batch
    if len(w.batch) > 0 {
        w.writer.Write(w.batch)
        w.count += len(w.batch)
        w.batch = w.batch[:0]
    }
    // THIS creates a new row group every time
    if err := w.writer.Flush(); err != nil {
        return fmt.Errorf("flushing parquet row group: %w", err)
    }
    return nil
}
```

## Design constraints

- The parquet-go library hard-limits row groups to 32767 per file
- We want good compression (data sorted by cluster, hostname, metric, timestamp)
- Don't accumulate all rows in memory — streaming is important
- Performance: this is HPC monitoring at scale

## Possible approaches

1. **Remove per-checkpoint Flush, flush periodically instead**: Instead of flushing after every checkpoint file, flush after accumulating a certain number of rows (e.g., every 100k rows or every N checkpoints). This reduces row groups dramatically while still bounding memory.

2. **Split into multiple parquet files**: When approaching the row group limit, close the current file and open a new one.

3. **Flush per host instead of per checkpoint**: Since multiple checkpoints can belong to the same host, only flush once per host rather than per checkpoint file.

Which approach is best? Consider that:
- A host may have 1 to ~hundreds of checkpoint files
- A cluster may have ~100 to ~3000+ hosts
- Flushing per host would give at most ~3000 row groups (within limit for most clusters)
- But combining approaches 1+2 would be most robust

Please design the recommended implementation approach with specific code changes needed.


Write

# Fix: Parquet Row Group Limit in Metric Store Archiving

## Context

The cleanup archiver creates one parquet file per cluster and calls `writer.Flush()` after every checkpoint file, creating one row group per checkpoint. The parquet-go library hard-limits row groups to 32767 per file. Large clusters like "fritz" with many hosts and multiple checkpoint files per host exceed this limit, causing: `flushing parquet row group: the limit of 32767 row groups has been reached`

## Root Cause

In `WriteCheckpointFile()` (`parquetArchive.go:115`), `w.writer.Flush()` is called after **every** checkpoint file. For a cluster with e.g. 1000 hosts × 40 checkpoint files = 40,000 flushes → 40,000 row groups → exceeds 32767.

## Fix

Move the `Flush()` out of `WriteCheckpointFile` and into the caller, flushing **once per host** instead of once per checkpoint file. This gives at most one row group per host (e.g. ~3000 for a large cluster), well within the limit.

This also improves compression — all checkpoint data for a single host is now in one row group with highly repetitive column values.

### File 1: `pkg/metricstore/parquetArchive.go`

1. **Remove `Flush()` from `WriteCheckpointFile`** (lines 115-117) — keep only the batch-drain logic
2. **Add `FlushRowGroup()` method** — wraps `w.writer.Flush()` for the caller to invoke explicitly

### File 2: `pkg/metricstore/archive.go`

3. **Call `FlushRowGroup()` once per host** in the `for r := range results` loop (after all checkpoint files for that host are written), instead of relying on per-checkpoint flushes:

```go
for r := range results {
    if writeErr == nil {
        for _, cf := range r.checkpoints {
            if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
                writeErr = err
                break
            }
        }
        if writeErr == nil {
            if err := writer.FlushRowGroup(); err != nil {
                writeErr = err
            }
        }
    }
    toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
}
```

## Verification

- `go build ./pkg/metricstore/...` — compiles
- `go test ./pkg/metricstore/...` — existing tests pass
- Manually verify: for a cluster with N hosts, the parquet file should contain N row groups (not N × checkpoints-per-host)


File created successfully at: /Users/jan/.claude/plans/agile-napping-otter.md

# Fix: Parquet Row Group Limit in Metric Store Archiving

## Context

The cleanup archiver creates one parquet file per cluster and calls `writer.Flush()` after every checkpoint file, creating one row group per checkpoint. The parquet-go library hard-limits row groups to 32767 per file. Large clusters like "fritz" with many hosts and multiple checkpoint files per host exceed this limit, causing: `flushing parquet row group: the limit of 32767 row groups has been reached`

## Root Cause

In `WriteCheckpointFile()` (`parquetArchive.go:115`), `w.writer.Flush()` is called after **every** checkpoint file. For a cluster with e.g. 1000 hosts × 40 checkpoint files = 40,000 flushes → 40,000 row groups → exceeds 32767.

## Fix

Move the `Flush()` out of `WriteCheckpointFile` and into the caller, flushing **once per host** instead of once per checkpoint file. This gives at most one row group per host (e.g. ~3000 for a large cluster), well within the limit.

This also improves compression — all checkpoint data for a single host is now in one row group with highly repetitive column values.

### File 1: `pkg/metricstore/parquetArchive.go`

1. **Remove `Flush()` from `WriteCheckpointFile`** (lines 115-117) — keep only the batch-drain logic
2. **Add `FlushRowGroup()` method** — wraps `w.writer.Flush()` for the caller to invoke explicitly

### File 2: `pkg/metricstore/archive.go`

3. **Call `FlushRowGroup()` once per host** in the `for r := range results` loop (after all checkpoint files for that host are written), instead of relying on per-checkpoint flushes:

```go
for r := range results {
    if writeErr == nil {
        for _, cf := range r.checkpoints {
            if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
                writeErr = err
                break
            }
        }
        if writeErr == nil {
            if err := writer.FlushRowGroup(); err != nil {
                writeErr = err
            }
        }
    }
    toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
}
```

## Verification

- `go build ./pkg/metricstore/...` — compiles
- `go test ./pkg/metricstore/...` — existing tests pass
- Manually verify: for a cluster with N hosts, the parquet file should contain N row groups (not N × checkpoints-per-host)


ToolSearch

Question: select:ExitPlanMode


Question: select:ExitPlanMode


ExitPlanMode

Prompt: run go build and go test for metricstore package

# Fix: Parquet Row Group Limit in Metric Store Archiving

## Context

The cleanup archiver creates one parquet file per cluster and calls `writer.Flush()` after every checkpoint file, creating one row group per checkpoint. The parquet-go library hard-limits row groups to 32767 per file. Large clusters like "fritz" with many hosts and multiple checkpoint files per host exceed this limit, causing: `flushing parquet row group: the limit of 32767 row groups has been reached`

## Root Cause

In `WriteCheckpointFile()` (`parquetArchive.go:115`), `w.writer.Flush()` is called after **every** checkpoint file. For a cluster with e.g. 1000 hosts × 40 checkpoint files = 40,000 flushes → 40,000 row groups → exceeds 32767.

## Fix

Move the `Flush()` out of `WriteCheckpointFile` and into the caller, flushing **once per host** instead of once per checkpoint file. This gives at most one row group per host (e.g. ~3000 for a large cluster), well within the limit.

This also improves compression — all checkpoint data for a single host is now in one row group with highly repetitive column values.

### File 1: `pkg/metricstore/parquetArchive.go`

1. **Remove `Flush()` from `WriteCheckpointFile`** (lines 115-117) — keep only the batch-drain logic
2. **Add `FlushRowGroup()` method** — wraps `w.writer.Flush()` for the caller to invoke explicitly

### File 2: `pkg/metricstore/archive.go`

3. **Call `FlushRowGroup()` once per host** in the `for r := range results` loop (after all checkpoint files for that host are written), instead of relying on per-checkpoint flushes:

```go
for r := range results {
    if writeErr == nil {
        for _, cf := range r.checkpoints {
            if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
                writeErr = err
                break
            }
        }
        if writeErr == nil {
            if err := writer.FlushRowGroup(); err != nil {
                writeErr = err
            }
        }
    }
    toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
}
```

## Verification

- `go build ./pkg/metricstore/...` — compiles
- `go test ./pkg/metricstore/...` — existing tests pass
- Manually verify: for a cluster with N hosts, the parquet file should contain N row groups (not N × checkpoints-per-host)


The user doesn't want to proceed with this tool use. The tool use was rejected (eg. if it was a file edit, the new_string was NOT written to the file). STOP what you are doing and wait for the user to tell you how to proceed.


[Request interrupted by user for tool use]