     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides config.go: Configuration structures and metric management.
     7→//
     8→// # Configuration Hierarchy
     9→//
    10→// The metricstore package uses nested configuration structures:
    11→//
    12→//	MetricStoreConfig (Keys)
    13→//	├─ NumWorkers: Parallel checkpoint/archive workers
    14→//	├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval)
    15→//	├─ MemoryCap: Memory limit in bytes (triggers forceFree)
    16→//	├─ Checkpoints: Persistence configuration
    17→//	│  ├─ FileFormat: "json" or "wal" (default: "wal")
    18→//	│  └─ RootDir: Checkpoint storage path
    19→//	├─ Cleanup: Long-term storage configuration (interval = RetentionInMemory)
    20→//	│  ├─ RootDir: Archive storage path (archive mode only)
    21→//	│  └─ Mode: "delete" or "archive"
    22→//	├─ Debug: Development/debugging options
    23→//	└─ Subscriptions: NATS topic subscriptions for metric ingestion
    24→//
    25→// # Metric Configuration
    26→//
    27→// Each metric (e.g., "cpu_load", "mem_used") has a MetricConfig entry in the global
    28→// Metrics map, defining:
    29→//
    30→//   - Frequency: Measurement interval in seconds
    31→//   - Aggregation: How to combine values (sum/avg/none) when transforming scopes
    32→//   - offset: Internal index into Level.metrics slice (assigned during Init)
    33→//
    34→// # AggregationStrategy
    35→//
    36→// Determines how to combine metric values when aggregating from finer to coarser scopes:
    37→//
    38→//   - NoAggregation: Do not combine (incompatible scopes)
    39→//   - SumAggregation: Add values (e.g., power consumption: core→socket)
    40→//   - AvgAggregation: Average values (e.g., temperature: core→socket)
    41→package metricstore
    42→
    43→import (
    44→	"fmt"
    45→	"time"
    46→
    47→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    48→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    49→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    50→)
    51→
    52→const (
    53→	DefaultMaxWorkers                 = 10
    54→	DefaultBufferCapacity             = 512
    55→	DefaultGCTriggerInterval          = 100
    56→	DefaultMemoryUsageTrackerInterval = 5 * time.Minute
    57→)
    58→
    59→// Checkpoints configures periodic persistence of in-memory metric data.
    60→//
    61→// Fields:
    62→//   - FileFormat:  "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal"
    63→//   - RootDir:     Filesystem path for checkpoint files (created if missing)
    64→//   - MaxWALSize:  Maximum size in bytes for a single host's WAL file; 0 = unlimited (default).
    65→//     When exceeded the WAL is force-rotated to prevent unbounded disk growth.
    66→type Checkpoints struct {
    67→	FileFormat string `json:"file-format"`
    68→	RootDir    string `json:"directory"`
    69→	MaxWALSize int64  `json:"max-wal-size,omitempty"`
    70→}
    71→
    72→// Debug provides development and profiling options.
    73→//
    74→// Fields:
    75→//   - DumpToFile: Path to dump checkpoint data for inspection (empty = disabled)
    76→//   - EnableGops: Enable gops agent for live runtime debugging (https://github.com/google/gops)
    77→type Debug struct {
    78→	DumpToFile string `json:"dump-to-file"`
    79→	EnableGops bool   `json:"gops"`
    80→}
    81→
    82→// Cleanup configures long-term storage of old metric data.
    83→//
    84→// Data older than RetentionInMemory is archived to disk or deleted.
    85→// The cleanup interval is always RetentionInMemory.
    86→//
    87→// Fields:
    88→//   - RootDir: Filesystem path for archived data (used in "archive" mode)
    89→//   - Mode:    "delete" (discard old data) or "archive" (write to RootDir)
    90→type Cleanup struct {
    91→	RootDir string `json:"directory"`
    92→	Mode    string `json:"mode"`
    93→}
    94→
    95→// Subscriptions defines NATS topics to subscribe to for metric ingestion.
    96→//
    97→// Each subscription receives metrics via NATS messaging, enabling real-time
    98→// data collection from compute nodes.
    99→//
   100→// Fields:
   101→//   - SubscribeTo: NATS subject/channel name (e.g., "metrics.compute.*")
   102→//   - ClusterTag:  Default cluster name for metrics without cluster tag (optional)
   103→type Subscriptions []struct {
   104→	// Channel name
   105→	SubscribeTo string `json:"subscribe-to"`
   106→
   107→	// Allow lines without a cluster tag, use this as default, optional
   108→	ClusterTag string `json:"cluster-tag"`
   109→}
   110→
   111→// MetricStoreConfig defines the main configuration for the metricstore.
   112→//
   113→// Loaded from cc-backend's config.json "metricstore" section. Controls memory usage,
   114→// persistence, archiving, and metric ingestion.
   115→//
   116→// Fields:
   117→//   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
   118→//   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
   119→//   - MemoryCap:         Max memory in GB for buffer data (0 = unlimited); triggers forceFree when exceeded
   120→//   - Checkpoints:       Periodic persistence configuration
   121→//   - Debug:             Development/profiling options (nil = disabled)
   122→//   - Archive:           Long-term storage configuration (nil = disabled)
   123→//   - Subscriptions:     NATS topics for metric ingestion (nil = polling only)
   124→type MetricStoreConfig struct {
   125→	// Number of concurrent workers for checkpoint and archive operations.
   126→	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
   127→	NumWorkers         int            `json:"num-workers"`
   128→	RetentionInMemory  string         `json:"retention-in-memory"`
   129→	CheckpointInterval string         `json:"checkpoint-interval,omitempty"`
   130→	MemoryCap          int            `json:"memory-cap"`
   131→	Checkpoints        Checkpoints    `json:"checkpoints"`
   132→	Debug              *Debug         `json:"debug"`
   133→	Cleanup            *Cleanup       `json:"cleanup"`
   134→	Subscriptions      *Subscriptions `json:"nats-subscriptions"`
   135→}
   136→
   137→// Keys is the global metricstore configuration instance.
   138→//
   139→// Initialized with defaults, then overwritten by cc-backend's config.json.
   140→// Accessed by Init(), Checkpointing(), and other lifecycle functions.
   141→var Keys MetricStoreConfig = MetricStoreConfig{
   142→	Checkpoints: Checkpoints{
   143→		FileFormat: "wal",
   144→		RootDir:    "./var/checkpoints",
   145→	},
   146→	Cleanup: &Cleanup{
   147→		Mode: "delete",
   148→	},
   149→}
   150→
   151→// AggregationStrategy defines how to combine metric values across hierarchy levels.
   152→//
   153→// Used when transforming data from finer-grained scopes (e.g., core) to coarser scopes
   154→// (e.g., socket). This is SPATIAL aggregation, not TEMPORAL (time-based) aggregation.
   155→//
   156→// Values:
   157→//   - NoAggregation:  Do not aggregate (incompatible scopes or non-aggregatable metrics)
   158→//   - SumAggregation: Add values (e.g., power: sum core power → socket power)
   159→//   - AvgAggregation: Average values (e.g., temperature: average core temps → socket temp)
   160→type AggregationStrategy int
   161→
   162→const (
   163→	NoAggregation  AggregationStrategy = iota // Do not aggregate
   164→	SumAggregation                            // Sum values (e.g., power, energy)
   165→	AvgAggregation                            // Average values (e.g., temperature, utilization)
   166→)
   167→
   168→// AssignAggregationStrategy parses a string into an AggregationStrategy value.
   169→//
   170→// Used when loading metric configurations from JSON/YAML files.
   171→//
   172→// Parameters:
   173→//   - str: "sum", "avg", or "" (empty string for NoAggregation)
   174→//
   175→// Returns:
   176→//   - AggregationStrategy: Parsed value
   177→//   - error:               Non-nil if str is unrecognized
   178→func AssignAggregationStrategy(str string) (AggregationStrategy, error) {
   179→	switch str {
   180→	case "":
   181→		return NoAggregation, nil
   182→	case "sum":
   183→		return SumAggregation, nil
   184→	case "avg":
   185→		return AvgAggregation, nil
   186→	default:
   187→		return NoAggregation, fmt.Errorf("[METRICSTORE]> unknown aggregation strategy: %s", str)
   188→	}
   189→}
   190→
   191→// MetricConfig defines configuration for a single metric type.
   192→//
   193→// Stored in the global Metrics map, keyed by metric name (e.g., "cpu_load").
   194→//
   195→// Fields:
   196→//   - Frequency:   Measurement interval in seconds (e.g., 60 for 1-minute granularity)
   197→//   - Aggregation: How to combine values across hierarchy levels (sum/avg/none)
   198→//   - offset:      Internal index into Level.metrics slice (assigned during Init)
   199→type MetricConfig struct {
   200→	// Interval in seconds at which measurements are stored
   201→	Frequency int64
   202→
   203→	// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
   204→	Aggregation AggregationStrategy
   205→
   206→	// Private, used internally...
   207→	offset int
   208→}
   209→
   210→func BuildMetricList() map[string]MetricConfig {
   211→	var metrics map[string]MetricConfig = make(map[string]MetricConfig)
   212→
   213→	addMetric := func(name string, metric MetricConfig) error {
   214→		if metrics == nil {
   215→			metrics = make(map[string]MetricConfig, 0)
   216→		}
   217→
   218→		if existingMetric, ok := metrics[name]; ok {
   219→			if existingMetric.Frequency != metric.Frequency {
   220→				if existingMetric.Frequency < metric.Frequency {
   221→					existingMetric.Frequency = metric.Frequency
   222→					metrics[name] = existingMetric
   223→				}
   224→			}
   225→		} else {
   226→			metrics[name] = metric
   227→		}
   228→
   229→		return nil
   230→	}
   231→
   232→	// Helper function to add metric configuration
   233→	addMetricConfig := func(mc *schema.MetricConfig) {
   234→		agg, err := AssignAggregationStrategy(mc.Aggregation)
   235→		if err != nil {
   236→			cclog.Warnf("Could not find aggregation strategy for metric config '%s': %s", mc.Name, err.Error())
   237→		}
   238→
   239→		addMetric(mc.Name, MetricConfig{
   240→			Frequency:   int64(mc.Timestep),
   241→			Aggregation: agg,
   242→		})
   243→	}
   244→	for _, c := range archive.Clusters {
   245→		for _, mc := range c.MetricConfig {
   246→			addMetricConfig(mc)
   247→		}
   248→
   249→		for _, sc := range c.SubClusters {
   250→			for _, mc := range sc.MetricConfig {
   251→				addMetricConfig(mc)
   252→			}
   253→		}
   254→	}
   255→
   256→	return metrics
   257→}
   258→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides config.go: Configuration structures and metric management.
//
// # Configuration Hierarchy
//
// The metricstore package uses nested configuration structures:
//
//	MetricStoreConfig (Keys)
//	├─ NumWorkers: Parallel checkpoint/archive workers
//	├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval)
//	├─ MemoryCap: Memory limit in bytes (triggers forceFree)
//	├─ Checkpoints: Persistence configuration
//	│  ├─ FileFormat: "json" or "wal" (default: "wal")
//	│  └─ RootDir: Checkpoint storage path
//	├─ Cleanup: Long-term storage configuration (interval = RetentionInMemory)
//	│  ├─ RootDir: Archive storage path (archive mode only)
//	│  └─ Mode: "delete" or "archive"
//	├─ Debug: Development/debugging options
//	└─ Subscriptions: NATS topic subscriptions for metric ingestion
//
// # Metric Configuration
//
// Each metric (e.g., "cpu_load", "mem_used") has a MetricConfig entry in the global
// Metrics map, defining:
//
//   - Frequency: Measurement interval in seconds
//   - Aggregation: How to combine values (sum/avg/none) when transforming scopes
//   - offset: Internal index into Level.metrics slice (assigned during Init)
//
// # AggregationStrategy
//
// Determines how to combine metric values when aggregating from finer to coarser scopes:
//
//   - NoAggregation: Do not combine (incompatible scopes)
//   - SumAggregation: Add values (e.g., power consumption: core→socket)
//   - AvgAggregation: Average values (e.g., temperature: core→socket)
package metricstore

import (
	"fmt"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

const (
	DefaultMaxWorkers                 = 10
	DefaultBufferCapacity             = 512
	DefaultGCTriggerInterval          = 100
	DefaultMemoryUsageTrackerInterval = 5 * time.Minute
)

// Checkpoints configures periodic persistence of in-memory metric data.
//
// Fields:
//   - FileFormat:  "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal"
//   - RootDir:     Filesystem path for checkpoint files (created if missing)
//   - MaxWALSize:  Maximum size in bytes for a single host's WAL file; 0 = unlimited (default).
//     When exceeded the WAL is force-rotated to prevent unbounded disk growth.
type Checkpoints struct {
	FileFormat string `json:"file-format"`
	RootDir    string `json:"directory"`
	MaxWALSize int64  `json:"max-wal-size,omitempty"`
}

// Debug provides development and profiling options.
//
// Fields:
//   - DumpToFile: Path to dump checkpoint data for inspection (empty = disabled)
//   - EnableGops: Enable gops agent for live runtime debugging (https://github.com/google/gops)
type Debug struct {
	DumpToFile string `json:"dump-to-file"`
	EnableGops bool   `json:"gops"`
}

// Cleanup configures long-term storage of old metric data.
//
// Data older than RetentionInMemory is archived to disk or deleted.
// The cleanup interval is always RetentionInMemory.
//
// Fields:
//   - RootDir: Filesystem path for archived data (used in "archive" mode)
//   - Mode:    "delete" (discard old data) or "archive" (write to RootDir)
type Cleanup struct {
	RootDir string `json:"directory"`
	Mode    string `json:"mode"`
}

// Subscriptions defines NATS topics to subscribe to for metric ingestion.
//
// Each subscription receives metrics via NATS messaging, enabling real-time
// data collection from compute nodes.
//
// Fields:
//   - SubscribeTo: NATS subject/channel name (e.g., "metrics.compute.*")
//   - ClusterTag:  Default cluster name for metrics without cluster tag (optional)
type Subscriptions []struct {
	// Channel name
	SubscribeTo string `json:"subscribe-to"`

	// Allow lines without a cluster tag, use this as default, optional
	ClusterTag string `json:"cluster-tag"`
}

// MetricStoreConfig defines the main configuration for the metricstore.
//
// Loaded from cc-backend's config.json "metricstore" section. Controls memory usage,
// persistence, archiving, and metric ingestion.
//
// Fields:
//   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
//   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
//   - MemoryCap:         Max memory in GB for buffer data (0 = unlimited); triggers forceFree when exceeded
//   - Checkpoints:       Periodic persistence configuration
//   - Debug:             Development/profiling options (nil = disabled)
//   - Archive:           Long-term storage configuration (nil = disabled)
//   - Subscriptions:     NATS topics for metric ingestion (nil = polling only)
type MetricStoreConfig struct {
	// Number of concurrent workers for checkpoint and archive operations.
	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
	NumWorkers         int            `json:"num-workers"`
	RetentionInMemory  string         `json:"retention-in-memory"`
	CheckpointInterval string         `json:"checkpoint-interval,omitempty"`
	MemoryCap          int            `json:"memory-cap"`
	Checkpoints        Checkpoints    `json:"checkpoints"`
	Debug              *Debug         `json:"debug"`
	Cleanup            *Cleanup       `json:"cleanup"`
	Subscriptions      *Subscriptions `json:"nats-subscriptions"`
}

// Keys is the global metricstore configuration instance.
//
// Initialized with defaults, then overwritten by cc-backend's config.json.
// Accessed by Init(), Checkpointing(), and other lifecycle functions.
var Keys MetricStoreConfig = MetricStoreConfig{
	Checkpoints: Checkpoints{
		FileFormat: "wal",
		RootDir:    "./var/checkpoints",
	},
	Cleanup: &Cleanup{
		Mode: "delete",
	},
}

// AggregationStrategy defines how to combine metric values across hierarchy levels.
//
// Used when transforming data from finer-grained scopes (e.g., core) to coarser scopes
// (e.g., socket). This is SPATIAL aggregation, not TEMPORAL (time-based) aggregation.
//
// Values:
//   - NoAggregation:  Do not aggregate (incompatible scopes or non-aggregatable metrics)
//   - SumAggregation: Add values (e.g., power: sum core power → socket power)
//   - AvgAggregation: Average values (e.g., temperature: average core temps → socket temp)
type AggregationStrategy int

const (
	NoAggregation  AggregationStrategy = iota // Do not aggregate
	SumAggregation                            // Sum values (e.g., power, energy)
	AvgAggregation                            // Average values (e.g., temperature, utilization)
)

// AssignAggregationStrategy parses a string into an AggregationStrategy value.
//
// Used when loading metric configurations from JSON/YAML files.
//
// Parameters:
//   - str: "sum", "avg", or "" (empty string for NoAggregation)
//
// Returns:
//   - AggregationStrategy: Parsed value
//   - error:               Non-nil if str is unrecognized
func AssignAggregationStrategy(str string) (AggregationStrategy, error) {
	switch str {
	case "":
		return NoAggregation, nil
	case "sum":
		return SumAggregation, nil
	case "avg":
		return AvgAggregation, nil
	default:
		return NoAggregation, fmt.Errorf("[METRICSTORE]> unknown aggregation strategy: %s", str)
	}
}

// MetricConfig defines configuration for a single metric type.
//
// Stored in the global Metrics map, keyed by metric name (e.g., "cpu_load").
//
// Fields:
//   - Frequency:   Measurement interval in seconds (e.g., 60 for 1-minute granularity)
//   - Aggregation: How to combine values across hierarchy levels (sum/avg/none)
//   - offset:      Internal index into Level.metrics slice (assigned during Init)
type MetricConfig struct {
	// Interval in seconds at which measurements are stored
	Frequency int64

	// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
	Aggregation AggregationStrategy

	// Private, used internally...
	offset int
}

func BuildMetricList() map[string]MetricConfig {
	var metrics map[string]MetricConfig = make(map[string]MetricConfig)

	addMetric := func(name string, metric MetricConfig) error {
		if metrics == nil {
			metrics = make(map[string]MetricConfig, 0)
		}

		if existingMetric, ok := metrics[name]; ok {
			if existingMetric.Frequency != metric.Frequency {
				if existingMetric.Frequency < metric.Frequency {
					existingMetric.Frequency = metric.Frequency
					metrics[name] = existingMetric
				}
			}
		} else {
			metrics[name] = metric
		}

		return nil
	}

	// Helper function to add metric configuration
	addMetricConfig := func(mc *schema.MetricConfig) {
		agg, err := AssignAggregationStrategy(mc.Aggregation)
		if err != nil {
			cclog.Warnf("Could not find aggregation strategy for metric config '%s': %s", mc.Name, err.Error())
		}

		addMetric(mc.Name, MetricConfig{
			Frequency:   int64(mc.Timestep),
			Aggregation: agg,
		})
	}
	for _, c := range archive.Clusters {
		for _, mc := range c.MetricConfig {
			addMetricConfig(mc)
		}

		for _, sc := range c.SubClusters {
			for _, mc := range sc.MetricConfig {
				addMetricConfig(mc)
			}
		}
	}

	return metrics
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"context"
    10→	"errors"
    11→	"fmt"
    12→	"os"
    13→	"path/filepath"
    14→	"strings"
    15→	"sync"
    16→	"sync/atomic"
    17→	"time"
    18→
    19→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    20→)
    21→
    22→// Worker for either Archiving or Deleting files
    23→
    24→func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
    25→	if Keys.Cleanup.Mode == "archive" {
    26→		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
    27→		// Run as Archiver
    28→		cleanUpWorker(wg, ctx,
    29→			Keys.RetentionInMemory,
    30→			"archiving",
    31→			Keys.Cleanup.RootDir,
    32→			false,
    33→		)
    34→	} else {
    35→		// Run as Deleter
    36→		cleanUpWorker(wg, ctx,
    37→			Keys.RetentionInMemory,
    38→			"deleting",
    39→			"",
    40→			true,
    41→		)
    42→	}
    43→}
    44→
    45→// cleanUpWorker takes simple values to configure what it does
    46→func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
    47→	wg.Go(func() {
    48→		d, err := time.ParseDuration(interval)
    49→		if err != nil {
    50→			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
    51→		}
    52→		if d <= 0 {
    53→			return
    54→		}
    55→
    56→		ticker := time.NewTicker(d)
    57→		defer ticker.Stop()
    58→
    59→		for {
    60→			select {
    61→			case <-ctx.Done():
    62→				return
    63→			case <-ticker.C:
    64→				t := time.Now().Add(-d)
    65→				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))
    66→
    67→				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)
    68→
    69→				if err != nil {
    70→					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
    71→				} else {
    72→					if delete {
    73→						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
    74→					} else {
    75→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
    76→					}
    77→				}
    78→			}
    79→		}
    80→	})
    81→}
    82→
    83→var ErrNoNewArchiveData error = errors.New("all data already archived")
    84→
    85→// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
    86→// When archiving, consolidates all hosts per cluster into a single Parquet file.
    87→func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
    88→	if deleteInstead {
    89→		return deleteCheckpoints(checkpointsDir, from)
    90→	}
    91→
    92→	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
    93→}
    94→
    95→// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
    96→func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
    97→	entries1, err := os.ReadDir(checkpointsDir)
    98→	if err != nil {
    99→		return 0, err
   100→	}
   101→
   102→	type workItem struct {
   103→		dir           string
   104→		cluster, host string
   105→	}
   106→
   107→	var wg sync.WaitGroup
   108→	n, errs := int32(0), int32(0)
   109→	work := make(chan workItem, Keys.NumWorkers)
   110→
   111→	wg.Add(Keys.NumWorkers)
   112→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   113→		go func() {
   114→			defer wg.Done()
   115→			for item := range work {
   116→				entries, err := os.ReadDir(item.dir)
   117→				if err != nil {
   118→					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
   119→					atomic.AddInt32(&errs, 1)
   120→					continue
   121→				}
   122→
   123→				files, err := findFiles(entries, from, false)
   124→				if err != nil {
   125→					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
   126→					atomic.AddInt32(&errs, 1)
   127→					continue
   128→				}
   129→
   130→				for _, checkpoint := range files {
   131→					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
   132→						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
   133→						atomic.AddInt32(&errs, 1)
   134→					} else {
   135→						atomic.AddInt32(&n, 1)
   136→					}
   137→				}
   138→			}
   139→		}()
   140→	}
   141→
   142→	for _, de1 := range entries1 {
   143→		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
   144→		if e != nil {
   145→			err = e
   146→			continue
   147→		}
   148→
   149→		for _, de2 := range entries2 {
   150→			work <- workItem{
   151→				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
   152→				cluster: de1.Name(),
   153→				host:    de2.Name(),
   154→			}
   155→		}
   156→	}
   157→
   158→	close(work)
   159→	wg.Wait()
   160→
   161→	if err != nil {
   162→		return int(n), err
   163→	}
   164→	if errs > 0 {
   165→		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
   166→	}
   167→	return int(n), nil
   168→}
   169→
   170→// archiveCheckpoints archives checkpoint files to Parquet format.
   171→// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
   172→// Workers load checkpoint files from disk and send CheckpointFile trees on a
   173→// back-pressured channel. The main thread streams each tree directly to Parquet
   174→// rows without materializing all rows in memory.
   175→func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
   176→	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
   177→	startTime := time.Now()
   178→
   179→	clusterEntries, err := os.ReadDir(checkpointsDir)
   180→	if err != nil {
   181→		return 0, err
   182→	}
   183→
   184→	totalFiles := 0
   185→	var clusterErrors []string
   186→
   187→	for _, clusterEntry := range clusterEntries {
   188→		if !clusterEntry.IsDir() {
   189→			continue
   190→		}
   191→
   192→		cluster := clusterEntry.Name()
   193→		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
   194→		if err != nil {
   195→			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
   196→			clusterErrors = append(clusterErrors, cluster)
   197→			continue
   198→		}
   199→
   200→		// Workers load checkpoint files from disk; main thread writes to parquet.
   201→		type hostResult struct {
   202→			checkpoints []*CheckpointFile
   203→			hostname    string
   204→			files       []string // checkpoint filenames to delete after successful write
   205→			dir         string   // checkpoint directory for this host
   206→		}
   207→
   208→		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
   209→		results := make(chan hostResult, 2)
   210→		work := make(chan struct {
   211→			dir, host string
   212→		}, Keys.NumWorkers)
   213→
   214→		var wg sync.WaitGroup
   215→		errs := int32(0)
   216→
   217→		wg.Add(Keys.NumWorkers)
   218→		for w := 0; w < Keys.NumWorkers; w++ {
   219→			go func() {
   220→				defer wg.Done()
   221→				for item := range work {
   222→					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
   223→					if err != nil {
   224→						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
   225→						atomic.AddInt32(&errs, 1)
   226→						continue
   227→					}
   228→					if len(checkpoints) > 0 {
   229→						results <- hostResult{
   230→							checkpoints: checkpoints,
   231→							hostname:    item.host,
   232→							files:       files,
   233→							dir:         item.dir,
   234→						}
   235→					}
   236→				}
   237→			}()
   238→		}
   239→
   240→		go func() {
   241→			for _, hostEntry := range hostEntries {
   242→				if !hostEntry.IsDir() {
   243→					continue
   244→				}
   245→				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
   246→				work <- struct {
   247→					dir, host string
   248→				}{dir: dir, host: hostEntry.Name()}
   249→			}
   250→			close(work)
   251→			wg.Wait()
   252→			close(results)
   253→		}()
   254→
   255→		// Open streaming writer and write each host's checkpoint files as a row group
   256→		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
   257→		writer, err := newParquetArchiveWriter(parquetFile)
   258→		if err != nil {
   259→			// Drain results channel to unblock workers
   260→			for range results {
   261→			}
   262→			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
   263→			clusterErrors = append(clusterErrors, cluster)
   264→			continue
   265→		}
   266→
   267→		type deleteItem struct {
   268→			dir   string
   269→			files []string
   270→		}
   271→		var toDelete []deleteItem
   272→		writeErr := error(nil)
   273→
   274→		for r := range results {
   275→			if writeErr == nil {
   276→				// Stream each checkpoint file directly to parquet rows.
   277→				// Each checkpoint is processed and discarded before the next.
   278→				for _, cf := range r.checkpoints {
   279→					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
   280→						writeErr = err
   281→						break
   282→					}
   283→				}
   284→				// Flush once per host to keep row group count within parquet limits.
   285→				if writeErr == nil {
   286→					if err := writer.FlushRowGroup(); err != nil {
   287→						writeErr = err
   288→					}
   289→				}
   290→			}
   291→			// Always track files for deletion (even if write failed, we still drain)
   292→			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
   293→		}
   294→
   295→		if err := writer.Close(); err != nil && writeErr == nil {
   296→			writeErr = err
   297→		}
   298→
   299→		if errs > 0 {
   300→			cclog.Errorf("[METRICSTORE]> %d errors reading checkpoints for cluster %s", errs, cluster)
   301→			clusterErrors = append(clusterErrors, cluster)
   302→			os.Remove(parquetFile)
   303→			continue
   304→		}
   305→
   306→		if writer.count == 0 {
   307→			// No data written — remove empty file
   308→			os.Remove(parquetFile)
   309→			continue
   310→		}
   311→
   312→		if writeErr != nil {
   313→			os.Remove(parquetFile)
   314→			cclog.Errorf("[METRICSTORE]> error writing parquet archive for cluster %s: %s", cluster, writeErr.Error())
   315→			clusterErrors = append(clusterErrors, cluster)
   316→			continue
   317→		}
   318→
   319→		// Delete archived checkpoint files
   320→		for _, item := range toDelete {
   321→			for _, file := range item.files {
   322→				filename := filepath.Join(item.dir, file)
   323→				if err := os.Remove(filename); err != nil {
   324→					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
   325→				} else {
   326→					totalFiles++
   327→				}
   328→			}
   329→		}
   330→
   331→		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
   332→			writer.count, totalFiles, cluster, parquetFile)
   333→	}
   334→
   335→	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)
   336→
   337→	if len(clusterErrors) > 0 {
   338→		return totalFiles, fmt.Errorf("archiving failed for clusters: %s", strings.Join(clusterErrors, ", "))
   339→	}
   340→
   341→	return totalFiles, nil
   342→}
   343→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
)

// Worker for either Archiving or Deleting files

func CleanUp(wg *sync.WaitGroup, ctx context.Context) {
	if Keys.Cleanup.Mode == "archive" {
		cclog.Info("[METRICSTORE]> enable archive cleanup to parquet")
		// Run as Archiver
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"archiving",
			Keys.Cleanup.RootDir,
			false,
		)
	} else {
		// Run as Deleter
		cleanUpWorker(wg, ctx,
			Keys.RetentionInMemory,
			"deleting",
			"",
			true,
		)
	}
}

// cleanUpWorker takes simple values to configure what it does
func cleanUpWorker(wg *sync.WaitGroup, ctx context.Context, interval string, mode string, cleanupDir string, delete bool) {
	wg.Go(func() {
		d, err := time.ParseDuration(interval)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> error parsing %s interval duration: %v\n", mode, err)
		}
		if d <= 0 {
			return
		}

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				t := time.Now().Add(-d)
				cclog.Infof("[METRICSTORE]> start %s checkpoints (older than %s)...", mode, t.Format(time.RFC3339))

				n, err := CleanupCheckpoints(Keys.Checkpoints.RootDir, cleanupDir, t.Unix(), delete)

				if err != nil {
					cclog.Errorf("[METRICSTORE]> %s failed: %s", mode, err.Error())
				} else {
					if delete {
						cclog.Infof("[METRICSTORE]> done: %d checkpoints deleted", n)
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files archived to parquet", n)
					}
				}
			}
		}
	})
}

var ErrNoNewArchiveData error = errors.New("all data already archived")

// CleanupCheckpoints deletes or archives all checkpoint files older than `from`.
// When archiving, consolidates all hosts per cluster into a single Parquet file.
func CleanupCheckpoints(checkpointsDir, cleanupDir string, from int64, deleteInstead bool) (int, error) {
	if deleteInstead {
		return deleteCheckpoints(checkpointsDir, from)
	}

	return archiveCheckpoints(checkpointsDir, cleanupDir, from)
}

// deleteCheckpoints removes checkpoint files older than `from` across all clusters/hosts.
func deleteCheckpoints(checkpointsDir string, from int64) (int, error) {
	entries1, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	type workItem struct {
		dir           string
		cluster, host string
	}

	var wg sync.WaitGroup
	n, errs := int32(0), int32(0)
	work := make(chan workItem, Keys.NumWorkers)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for item := range work {
				entries, err := os.ReadDir(item.dir)
				if err != nil {
					cclog.Errorf("error reading %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				files, err := findFiles(entries, from, false)
				if err != nil {
					cclog.Errorf("error finding files in %s/%s: %s", item.cluster, item.host, err.Error())
					atomic.AddInt32(&errs, 1)
					continue
				}

				for _, checkpoint := range files {
					if err := os.Remove(filepath.Join(item.dir, checkpoint)); err != nil {
						cclog.Errorf("error deleting %s/%s/%s: %s", item.cluster, item.host, checkpoint, err.Error())
						atomic.AddInt32(&errs, 1)
					} else {
						atomic.AddInt32(&n, 1)
					}
				}
			}
		}()
	}

	for _, de1 := range entries1 {
		entries2, e := os.ReadDir(filepath.Join(checkpointsDir, de1.Name()))
		if e != nil {
			err = e
			continue
		}

		for _, de2 := range entries2 {
			work <- workItem{
				dir:     filepath.Join(checkpointsDir, de1.Name(), de2.Name()),
				cluster: de1.Name(),
				host:    de2.Name(),
			}
		}
	}

	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}
	if errs > 0 {
		return int(n), fmt.Errorf("%d errors happened while deleting (%d successes)", errs, n)
	}
	return int(n), nil
}

// archiveCheckpoints archives checkpoint files to Parquet format.
// Produces one Parquet file per cluster: <cleanupDir>/<cluster>/<timestamp>.parquet
// Workers load checkpoint files from disk and send CheckpointFile trees on a
// back-pressured channel. The main thread streams each tree directly to Parquet
// rows without materializing all rows in memory.
func archiveCheckpoints(checkpointsDir, cleanupDir string, from int64) (int, error) {
	cclog.Info("[METRICSTORE]> start archiving checkpoints to parquet")
	startTime := time.Now()

	clusterEntries, err := os.ReadDir(checkpointsDir)
	if err != nil {
		return 0, err
	}

	totalFiles := 0
	var clusterErrors []string

	for _, clusterEntry := range clusterEntries {
		if !clusterEntry.IsDir() {
			continue
		}

		cluster := clusterEntry.Name()
		hostEntries, err := os.ReadDir(filepath.Join(checkpointsDir, cluster))
		if err != nil {
			cclog.Errorf("[METRICSTORE]> error reading host entries for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		// Workers load checkpoint files from disk; main thread writes to parquet.
		type hostResult struct {
			checkpoints []*CheckpointFile
			hostname    string
			files       []string // checkpoint filenames to delete after successful write
			dir         string   // checkpoint directory for this host
		}

		// Small buffer provides back-pressure: at most NumWorkers+2 results in flight.
		results := make(chan hostResult, 2)
		work := make(chan struct {
			dir, host string
		}, Keys.NumWorkers)

		var wg sync.WaitGroup
		errs := int32(0)

		wg.Add(Keys.NumWorkers)
		for w := 0; w < Keys.NumWorkers; w++ {
			go func() {
				defer wg.Done()
				for item := range work {
					checkpoints, files, err := loadCheckpointFiles(item.dir, from)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error reading checkpoints for %s/%s: %s", cluster, item.host, err.Error())
						atomic.AddInt32(&errs, 1)
						continue
					}
					if len(checkpoints) > 0 {
						results <- hostResult{
							checkpoints: checkpoints,
							hostname:    item.host,
							files:       files,
							dir:         item.dir,
						}
					}
				}
			}()
		}

		go func() {
			for _, hostEntry := range hostEntries {
				if !hostEntry.IsDir() {
					continue
				}
				dir := filepath.Join(checkpointsDir, cluster, hostEntry.Name())
				work <- struct {
					dir, host string
				}{dir: dir, host: hostEntry.Name()}
			}
			close(work)
			wg.Wait()
			close(results)
		}()

		// Open streaming writer and write each host's checkpoint files as a row group
		parquetFile := filepath.Join(cleanupDir, cluster, fmt.Sprintf("%d.parquet", from))
		writer, err := newParquetArchiveWriter(parquetFile)
		if err != nil {
			// Drain results channel to unblock workers
			for range results {
			}
			cclog.Errorf("[METRICSTORE]> error creating parquet writer for cluster %s: %s", cluster, err.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		type deleteItem struct {
			dir   string
			files []string
		}
		var toDelete []deleteItem
		writeErr := error(nil)

		for r := range results {
			if writeErr == nil {
				// Stream each checkpoint file directly to parquet rows.
				// Each checkpoint is processed and discarded before the next.
				for _, cf := range r.checkpoints {
					if err := writer.WriteCheckpointFile(cf, cluster, r.hostname, "node", ""); err != nil {
						writeErr = err
						break
					}
				}
				// Flush once per host to keep row group count within parquet limits.
				if writeErr == nil {
					if err := writer.FlushRowGroup(); err != nil {
						writeErr = err
					}
				}
			}
			// Always track files for deletion (even if write failed, we still drain)
			toDelete = append(toDelete, deleteItem{dir: r.dir, files: r.files})
		}

		if err := writer.Close(); err != nil && writeErr == nil {
			writeErr = err
		}

		if errs > 0 {
			cclog.Errorf("[METRICSTORE]> %d errors reading checkpoints for cluster %s", errs, cluster)
			clusterErrors = append(clusterErrors, cluster)
			os.Remove(parquetFile)
			continue
		}

		if writer.count == 0 {
			// No data written — remove empty file
			os.Remove(parquetFile)
			continue
		}

		if writeErr != nil {
			os.Remove(parquetFile)
			cclog.Errorf("[METRICSTORE]> error writing parquet archive for cluster %s: %s", cluster, writeErr.Error())
			clusterErrors = append(clusterErrors, cluster)
			continue
		}

		// Delete archived checkpoint files
		for _, item := range toDelete {
			for _, file := range item.files {
				filename := filepath.Join(item.dir, file)
				if err := os.Remove(filename); err != nil {
					cclog.Warnf("[METRICSTORE]> could not remove archived checkpoint %s: %v", filename, err)
				} else {
					totalFiles++
				}
			}
		}

		cclog.Infof("[METRICSTORE]> archived %d rows from %d files for cluster %s to %s",
			writer.count, totalFiles, cluster, parquetFile)
	}

	cclog.Infof("[METRICSTORE]> archiving checkpoints completed in %s (%d files)", time.Since(startTime).Round(time.Millisecond), totalFiles)

	if len(clusterErrors) > 0 {
		return totalFiles, fmt.Errorf("archiving failed for clusters: %s", strings.Join(clusterErrors, ", "))
	}

	return totalFiles, nil
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"encoding/json"
    10→	"fmt"
    11→	"time"
    12→
    13→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    14→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    15→)
    16→
    17→// HealthCheckResponse represents the result of a health check operation.
    18→type HealthCheckResponse struct {
    19→	Status schema.MonitoringState
    20→	Error  error
    21→}
    22→
    23→// HealthCheckResult holds the monitoring state and raw JSON health metrics
    24→// for a single node as determined by HealthCheck.
    25→type HealthCheckResult struct {
    26→	State         schema.MonitoringState
    27→	HealthMetrics string // JSON: {"missing":[...],"degraded":[...]}
    28→}
    29→
    30→// MaxMissingDataPoints is the threshold for stale data detection.
    31→// A buffer is considered healthy if the gap between its last data point
    32→// and the current time is within MaxMissingDataPoints * frequency.
    33→const MaxMissingDataPoints int64 = 5
    34→
    35→// bufferExists returns true if the buffer is non-nil and contains data.
    36→func (b *buffer) bufferExists() bool {
    37→	if b == nil || b.data == nil || len(b.data) == 0 {
    38→		return false
    39→	}
    40→
    41→	return true
    42→}
    43→
    44→// isBufferHealthy returns true if the buffer has recent data within
    45→// MaxMissingDataPoints * frequency of the current time.
    46→func (b *buffer) isBufferHealthy() bool {
    47→	bufferEnd := b.start + b.frequency*int64(len(b.data))
    48→	t := time.Now().Unix()
    49→
    50→	return t-bufferEnd <= MaxMissingDataPoints*b.frequency
    51→}
    52→
    53→// collectMetricStatus walks the subtree rooted at l and classifies each
    54→// expected metric into the healthy or degraded map.
    55→//
    56→// Classification rules (evaluated per buffer, pessimistic):
    57→//   - A single stale buffer marks the metric as degraded permanently.
    58→//   - A healthy buffer only counts if no stale buffer has been seen.
    59→//   - Metrics absent from the global config or without any buffer remain
    60→//     in neither map and are later reported as missing.
    61→func (l *Level) collectMetricStatus(m *MemoryStore, expectedMetrics []string, healthy, degraded map[string]bool) {
    62→	l.lock.RLock()
    63→	defer l.lock.RUnlock()
    64→
    65→	for _, metricName := range expectedMetrics {
    66→		if degraded[metricName] {
    67→			continue // already degraded, cannot improve
    68→		}
    69→		mc, ok := m.Metrics[metricName]
    70→		if !ok {
    71→			continue // unknown metric, will be reported as missing
    72→		}
    73→		b := l.metrics[mc.offset]
    74→		if b.bufferExists() {
    75→			if !b.isBufferHealthy() {
    76→				degraded[metricName] = true
    77→				delete(healthy, metricName)
    78→			} else if !degraded[metricName] {
    79→				healthy[metricName] = true
    80→			}
    81→		}
    82→	}
    83→
    84→	for _, lvl := range l.children {
    85→		lvl.collectMetricStatus(m, expectedMetrics, healthy, degraded)
    86→	}
    87→}
    88→
    89→// getHealthyMetrics walks the complete subtree rooted at l and classifies
    90→// each expected metric by comparing the collected status against the
    91→// expected list.
    92→//
    93→// Returns:
    94→//   - missingList: metrics not found in global config or without any buffer
    95→//   - degradedList: metrics with at least one stale buffer in the subtree
    96→func (l *Level) getHealthyMetrics(m *MemoryStore, expectedMetrics []string) ([]string, []string) {
    97→	healthy := make(map[string]bool, len(expectedMetrics))
    98→	degraded := make(map[string]bool)
    99→
   100→	l.collectMetricStatus(m, expectedMetrics, healthy, degraded)
   101→
   102→	missingList := make([]string, 0)
   103→	degradedList := make([]string, 0)
   104→
   105→	for _, metricName := range expectedMetrics {
   106→		if healthy[metricName] {
   107→			continue
   108→		}
   109→
   110→		if degraded[metricName] {
   111→			degradedList = append(degradedList, metricName)
   112→		} else {
   113→			missingList = append(missingList, metricName)
   114→		}
   115→	}
   116→
   117→	return degradedList, missingList
   118→}
   119→
   120→// GetHealthyMetrics returns missing and degraded metric lists for a node.
   121→//
   122→// It walks the metric tree starting from the node identified by selector
   123→// and classifies each expected metric:
   124→//   - Missing: no buffer anywhere in the subtree, or metric not in global config
   125→//   - Degraded: at least one stale buffer exists in the subtree
   126→//
   127→// Metrics present in expectedMetrics but absent from both returned lists
   128→// are considered fully healthy.
   129→func (m *MemoryStore) GetHealthyMetrics(selector []string, expectedMetrics []string) ([]string, []string, error) {
   130→	lvl := m.root.findLevel(selector)
   131→	if lvl == nil {
   132→		return nil, nil, fmt.Errorf("[METRICSTORE]> GetHealthyMetrics: host not found: %#v", selector)
   133→	}
   134→
   135→	degradedList, missingList := lvl.getHealthyMetrics(m, expectedMetrics)
   136→	return degradedList, missingList, nil
   137→}
   138→
   139→type HealthCheckReq struct {
   140→	Cluster     string   `json:"cluster" example:"fritz"`
   141→	Nodes       []string `json:"nodes"`
   142→	MetricNames []string `json:"metric-names"`
   143→}
   144→
   145→// HealthCheck evaluates multiple nodes against a set of expected metrics
   146→// and returns a monitoring state per node.
   147→//
   148→// States:
   149→//   - MonitoringStateFull: all expected metrics are healthy
   150→//   - MonitoringStatePartial: some metrics are missing or degraded
   151→//   - MonitoringStateFailed: node not found, or no healthy metrics at all
   152→func (m *MemoryStore) HealthCheck(cluster string,
   153→	nodes []string, expectedMetrics []string,
   154→) (map[string]HealthCheckResult, error) {
   155→	results := make(map[string]HealthCheckResult, len(nodes))
   156→
   157→	for _, hostname := range nodes {
   158→		selector := []string{cluster, hostname}
   159→
   160→		degradedList, missingList, err := m.GetHealthyMetrics(selector, expectedMetrics)
   161→		if err != nil {
   162→			results[hostname] = HealthCheckResult{State: schema.MonitoringStateFailed}
   163→			continue
   164→		}
   165→
   166→		degradedCount := len(degradedList)
   167→		missingCount := len(missingList)
   168→
   169→		healthyCount := len(expectedMetrics) - degradedCount - missingCount
   170→
   171→		if degradedCount > 0 {
   172→			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "degraded metrics:", degradedList)
   173→		}
   174→		if missingCount > 0 {
   175→			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "missing metrics:", missingList)
   176→		}
   177→
   178→		var state schema.MonitoringState
   179→		switch {
   180→		case degradedCount == 0 && missingCount == 0:
   181→			state = schema.MonitoringStateFull
   182→		case healthyCount == 0:
   183→			state = schema.MonitoringStateFailed
   184→		default:
   185→			state = schema.MonitoringStatePartial
   186→		}
   187→
   188→		hm, _ := json.Marshal(map[string][]string{
   189→			"missing":  missingList,
   190→			"degraded": degradedList,
   191→		})
   192→
   193→		results[hostname] = HealthCheckResult{
   194→			State:         state,
   195→			HealthMetrics: string(hm),
   196→		}
   197→	}
   198→
   199→	return results, nil
   200→}
   201→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"encoding/json"
	"fmt"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// HealthCheckResponse represents the result of a health check operation.
type HealthCheckResponse struct {
	Status schema.MonitoringState
	Error  error
}

// HealthCheckResult holds the monitoring state and raw JSON health metrics
// for a single node as determined by HealthCheck.
type HealthCheckResult struct {
	State         schema.MonitoringState
	HealthMetrics string // JSON: {"missing":[...],"degraded":[...]}
}

// MaxMissingDataPoints is the threshold for stale data detection.
// A buffer is considered healthy if the gap between its last data point
// and the current time is within MaxMissingDataPoints * frequency.
const MaxMissingDataPoints int64 = 5

// bufferExists returns true if the buffer is non-nil and contains data.
func (b *buffer) bufferExists() bool {
	if b == nil || b.data == nil || len(b.data) == 0 {
		return false
	}

	return true
}

// isBufferHealthy returns true if the buffer has recent data within
// MaxMissingDataPoints * frequency of the current time.
func (b *buffer) isBufferHealthy() bool {
	bufferEnd := b.start + b.frequency*int64(len(b.data))
	t := time.Now().Unix()

	return t-bufferEnd <= MaxMissingDataPoints*b.frequency
}

// collectMetricStatus walks the subtree rooted at l and classifies each
// expected metric into the healthy or degraded map.
//
// Classification rules (evaluated per buffer, pessimistic):
//   - A single stale buffer marks the metric as degraded permanently.
//   - A healthy buffer only counts if no stale buffer has been seen.
//   - Metrics absent from the global config or without any buffer remain
//     in neither map and are later reported as missing.
func (l *Level) collectMetricStatus(m *MemoryStore, expectedMetrics []string, healthy, degraded map[string]bool) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	for _, metricName := range expectedMetrics {
		if degraded[metricName] {
			continue // already degraded, cannot improve
		}
		mc, ok := m.Metrics[metricName]
		if !ok {
			continue // unknown metric, will be reported as missing
		}
		b := l.metrics[mc.offset]
		if b.bufferExists() {
			if !b.isBufferHealthy() {
				degraded[metricName] = true
				delete(healthy, metricName)
			} else if !degraded[metricName] {
				healthy[metricName] = true
			}
		}
	}

	for _, lvl := range l.children {
		lvl.collectMetricStatus(m, expectedMetrics, healthy, degraded)
	}
}

// getHealthyMetrics walks the complete subtree rooted at l and classifies
// each expected metric by comparing the collected status against the
// expected list.
//
// Returns:
//   - missingList: metrics not found in global config or without any buffer
//   - degradedList: metrics with at least one stale buffer in the subtree
func (l *Level) getHealthyMetrics(m *MemoryStore, expectedMetrics []string) ([]string, []string) {
	healthy := make(map[string]bool, len(expectedMetrics))
	degraded := make(map[string]bool)

	l.collectMetricStatus(m, expectedMetrics, healthy, degraded)

	missingList := make([]string, 0)
	degradedList := make([]string, 0)

	for _, metricName := range expectedMetrics {
		if healthy[metricName] {
			continue
		}

		if degraded[metricName] {
			degradedList = append(degradedList, metricName)
		} else {
			missingList = append(missingList, metricName)
		}
	}

	return degradedList, missingList
}

// GetHealthyMetrics returns missing and degraded metric lists for a node.
//
// It walks the metric tree starting from the node identified by selector
// and classifies each expected metric:
//   - Missing: no buffer anywhere in the subtree, or metric not in global config
//   - Degraded: at least one stale buffer exists in the subtree
//
// Metrics present in expectedMetrics but absent from both returned lists
// are considered fully healthy.
func (m *MemoryStore) GetHealthyMetrics(selector []string, expectedMetrics []string) ([]string, []string, error) {
	lvl := m.root.findLevel(selector)
	if lvl == nil {
		return nil, nil, fmt.Errorf("[METRICSTORE]> GetHealthyMetrics: host not found: %#v", selector)
	}

	degradedList, missingList := lvl.getHealthyMetrics(m, expectedMetrics)
	return degradedList, missingList, nil
}

type HealthCheckReq struct {
	Cluster     string   `json:"cluster" example:"fritz"`
	Nodes       []string `json:"nodes"`
	MetricNames []string `json:"metric-names"`
}

// HealthCheck evaluates multiple nodes against a set of expected metrics
// and returns a monitoring state per node.
//
// States:
//   - MonitoringStateFull: all expected metrics are healthy
//   - MonitoringStatePartial: some metrics are missing or degraded
//   - MonitoringStateFailed: node not found, or no healthy metrics at all
func (m *MemoryStore) HealthCheck(cluster string,
	nodes []string, expectedMetrics []string,
) (map[string]HealthCheckResult, error) {
	results := make(map[string]HealthCheckResult, len(nodes))

	for _, hostname := range nodes {
		selector := []string{cluster, hostname}

		degradedList, missingList, err := m.GetHealthyMetrics(selector, expectedMetrics)
		if err != nil {
			results[hostname] = HealthCheckResult{State: schema.MonitoringStateFailed}
			continue
		}

		degradedCount := len(degradedList)
		missingCount := len(missingList)

		healthyCount := len(expectedMetrics) - degradedCount - missingCount

		if degradedCount > 0 {
			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "degraded metrics:", degradedList)
		}
		if missingCount > 0 {
			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "missing metrics:", missingList)
		}

		var state schema.MonitoringState
		switch {
		case degradedCount == 0 && missingCount == 0:
			state = schema.MonitoringStateFull
		case healthyCount == 0:
			state = schema.MonitoringStateFailed
		default:
			state = schema.MonitoringStatePartial
		}

		hm, _ := json.Marshal(map[string][]string{
			"missing":  missingList,
			"degraded": degradedList,
		})

		results[hostname] = HealthCheckResult{
			State:         state,
			HealthMetrics: string(hm),
		}
	}

	return results, nil
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file contains shared scope transformation logic used by both the internal
     7→// metric store (pkg/metricstore) and the external cc-metric-store client
     8→// (internal/metricstoreclient). It extracts the common algorithm for mapping
     9→// between native metric scopes and requested scopes based on cluster topology.
    10→package metricstore
    11→
    12→import (
    13→	"strconv"
    14→
    15→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    16→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    17→)
    18→
    19→// Pre-converted scope strings avoid repeated string(MetricScope) allocations
    20→// during query construction. Used in ScopeQueryResult.Type field.
    21→var (
    22→	HWThreadString     = string(schema.MetricScopeHWThread)
    23→	CoreString         = string(schema.MetricScopeCore)
    24→	MemoryDomainString = string(schema.MetricScopeMemoryDomain)
    25→	SocketString       = string(schema.MetricScopeSocket)
    26→	AcceleratorString  = string(schema.MetricScopeAccelerator)
    27→)
    28→
    29→// ScopeQueryResult is a package-independent intermediate type returned by
    30→// BuildScopeQueries. Each consumer converts it to their own APIQuery type
    31→// (adding Resolution and any other package-specific fields).
    32→type ScopeQueryResult struct {
    33→	Type      *string
    34→	Metric    string
    35→	Hostname  string
    36→	TypeIds   []string
    37→	Scope     schema.MetricScope
    38→	Aggregate bool
    39→}
    40→
    41→// BuildScopeQueries generates scope query results for a given scope transformation.
    42→// It returns a slice of results and a boolean indicating success.
    43→// An empty slice means an expected exception (skip this combination).
    44→// ok=false means an unhandled case (caller should return an error).
    45→func BuildScopeQueries(
    46→	nativeScope, requestedScope schema.MetricScope,
    47→	metric, hostname string,
    48→	topology *schema.Topology,
    49→	hwthreads []int,
    50→	accelerators []string,
    51→) ([]ScopeQueryResult, bool) {
    52→	scope := nativeScope.Max(requestedScope)
    53→	results := []ScopeQueryResult{}
    54→
    55→	hwthreadsStr := IntToStringSlice(hwthreads)
    56→
    57→	// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
    58→	if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
    59→		if scope != schema.MetricScopeAccelerator {
    60→			// Expected Exception -> Return Empty Slice
    61→			return results, true
    62→		}
    63→
    64→		results = append(results, ScopeQueryResult{
    65→			Metric:    metric,
    66→			Hostname:  hostname,
    67→			Aggregate: false,
    68→			Type:      &AcceleratorString,
    69→			TypeIds:   accelerators,
    70→			Scope:     schema.MetricScopeAccelerator,
    71→		})
    72→		return results, true
    73→	}
    74→
    75→	// Accelerator -> Node
    76→	if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
    77→		if len(accelerators) == 0 {
    78→			// Expected Exception -> Return Empty Slice
    79→			return results, true
    80→		}
    81→
    82→		results = append(results, ScopeQueryResult{
    83→			Metric:    metric,
    84→			Hostname:  hostname,
    85→			Aggregate: true,
    86→			Type:      &AcceleratorString,
    87→			TypeIds:   accelerators,
    88→			Scope:     scope,
    89→		})
    90→		return results, true
    91→	}
    92→
    93→	// HWThread -> HWThread
    94→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
    95→		results = append(results, ScopeQueryResult{
    96→			Metric:    metric,
    97→			Hostname:  hostname,
    98→			Aggregate: false,
    99→			Type:      &HWThreadString,
   100→			TypeIds:   hwthreadsStr,
   101→			Scope:     scope,
   102→		})
   103→		return results, true
   104→	}
   105→
   106→	// HWThread -> Core
   107→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
   108→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   109→		for _, core := range cores {
   110→			results = append(results, ScopeQueryResult{
   111→				Metric:    metric,
   112→				Hostname:  hostname,
   113→				Aggregate: true,
   114→				Type:      &HWThreadString,
   115→				TypeIds:   IntToStringSlice(topology.Core[core]),
   116→				Scope:     scope,
   117→			})
   118→		}
   119→		return results, true
   120→	}
   121→
   122→	// HWThread -> Socket
   123→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
   124→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   125→		for _, socket := range sockets {
   126→			results = append(results, ScopeQueryResult{
   127→				Metric:    metric,
   128→				Hostname:  hostname,
   129→				Aggregate: true,
   130→				Type:      &HWThreadString,
   131→				TypeIds:   IntToStringSlice(topology.Socket[socket]),
   132→				Scope:     scope,
   133→			})
   134→		}
   135→		return results, true
   136→	}
   137→
   138→	// HWThread -> Node
   139→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
   140→		results = append(results, ScopeQueryResult{
   141→			Metric:    metric,
   142→			Hostname:  hostname,
   143→			Aggregate: true,
   144→			Type:      &HWThreadString,
   145→			TypeIds:   hwthreadsStr,
   146→			Scope:     scope,
   147→		})
   148→		return results, true
   149→	}
   150→
   151→	// Core -> Core
   152→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
   153→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   154→		results = append(results, ScopeQueryResult{
   155→			Metric:    metric,
   156→			Hostname:  hostname,
   157→			Aggregate: false,
   158→			Type:      &CoreString,
   159→			TypeIds:   IntToStringSlice(cores),
   160→			Scope:     scope,
   161→		})
   162→		return results, true
   163→	}
   164→
   165→	// Core -> Socket
   166→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
   167→		sockets, _ := topology.GetSocketsFromCores(hwthreads)
   168→		for _, socket := range sockets {
   169→			results = append(results, ScopeQueryResult{
   170→				Metric:    metric,
   171→				Hostname:  hostname,
   172→				Aggregate: true,
   173→				Type:      &CoreString,
   174→				TypeIds:   IntToStringSlice(topology.Socket[socket]),
   175→				Scope:     scope,
   176→			})
   177→		}
   178→		return results, true
   179→	}
   180→
   181→	// Core -> Node
   182→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
   183→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   184→		results = append(results, ScopeQueryResult{
   185→			Metric:    metric,
   186→			Hostname:  hostname,
   187→			Aggregate: true,
   188→			Type:      &CoreString,
   189→			TypeIds:   IntToStringSlice(cores),
   190→			Scope:     scope,
   191→		})
   192→		return results, true
   193→	}
   194→
   195→	// MemoryDomain -> MemoryDomain
   196→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
   197→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   198→		results = append(results, ScopeQueryResult{
   199→			Metric:    metric,
   200→			Hostname:  hostname,
   201→			Aggregate: false,
   202→			Type:      &MemoryDomainString,
   203→			TypeIds:   IntToStringSlice(memDomains),
   204→			Scope:     scope,
   205→		})
   206→		return results, true
   207→	}
   208→
   209→	// MemoryDomain -> Socket
   210→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket {
   211→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   212→		socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains)
   213→		if err != nil {
   214→			cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err)
   215→			// Rare Error Case -> Still Continue -> Return Empty Slice
   216→			return results, true
   217→		}
   218→
   219→		// Create a query for each socket
   220→		for _, domains := range socketToDomains {
   221→			results = append(results, ScopeQueryResult{
   222→				Metric:    metric,
   223→				Hostname:  hostname,
   224→				Aggregate: true,
   225→				Type:      &MemoryDomainString,
   226→				TypeIds:   IntToStringSlice(domains),
   227→				Scope:     scope,
   228→			})
   229→		}
   230→		return results, true
   231→	}
   232→
   233→	// MemoryDomain -> Node
   234→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
   235→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   236→		results = append(results, ScopeQueryResult{
   237→			Metric:    metric,
   238→			Hostname:  hostname,
   239→			Aggregate: true,
   240→			Type:      &MemoryDomainString,
   241→			TypeIds:   IntToStringSlice(memDomains),
   242→			Scope:     scope,
   243→		})
   244→		return results, true
   245→	}
   246→
   247→	// Socket -> Socket
   248→	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
   249→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   250→		results = append(results, ScopeQueryResult{
   251→			Metric:    metric,
   252→			Hostname:  hostname,
   253→			Aggregate: false,
   254→			Type:      &SocketString,
   255→			TypeIds:   IntToStringSlice(sockets),
   256→			Scope:     scope,
   257→		})
   258→		return results, true
   259→	}
   260→
   261→	// Socket -> Node
   262→	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
   263→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   264→		results = append(results, ScopeQueryResult{
   265→			Metric:    metric,
   266→			Hostname:  hostname,
   267→			Aggregate: true,
   268→			Type:      &SocketString,
   269→			TypeIds:   IntToStringSlice(sockets),
   270→			Scope:     scope,
   271→		})
   272→		return results, true
   273→	}
   274→
   275→	// Node -> Node
   276→	if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
   277→		results = append(results, ScopeQueryResult{
   278→			Metric:   metric,
   279→			Hostname: hostname,
   280→			Scope:    scope,
   281→		})
   282→		return results, true
   283→	}
   284→
   285→	// Unhandled Case
   286→	return nil, false
   287→}
   288→
   289→// IntToStringSlice converts a slice of integers to a slice of strings.
   290→// Used to convert hardware thread/core/socket IDs from topology (int) to query TypeIds (string).
   291→// Optimized to reuse a byte buffer for string conversion, reducing allocations.
   292→func IntToStringSlice(is []int) []string {
   293→	if len(is) == 0 {
   294→		return nil
   295→	}
   296→
   297→	ss := make([]string, len(is))
   298→	buf := make([]byte, 0, 16) // Reusable buffer for integer conversion
   299→	for i, x := range is {
   300→		buf = strconv.AppendInt(buf[:0], int64(x), 10)
   301→		ss[i] = string(buf)
   302→	}
   303→	return ss
   304→}
   305→
   306→// ExtractTypeID returns the type ID at the given index from a query's TypeIds slice.
   307→// Returns nil if queryType is nil (no type filtering). Logs a warning and returns nil
   308→// if the index is out of range.
   309→func ExtractTypeID(queryType *string, typeIds []string, ndx int, metric, hostname string) *string {
   310→	if queryType == nil {
   311→		return nil
   312→	}
   313→	if ndx < len(typeIds) {
   314→		id := typeIds[ndx]
   315→		return &id
   316→	}
   317→	cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   318→		ndx, len(typeIds), metric, hostname)
   319→	return nil
   320→}
   321→
   322→// IsMetricRemovedForSubCluster checks whether a metric is marked as removed
   323→// for the given subcluster in its per-subcluster configuration.
   324→func IsMetricRemovedForSubCluster(mc *schema.MetricConfig, subCluster string) bool {
   325→	for _, scConfig := range mc.SubClusters {
   326→		if scConfig.Name == subCluster && scConfig.Remove {
   327→			return true
   328→		}
   329→	}
   330→	return false
   331→}
   332→
   333→// SanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
   334→// If ANY of avg/min/max is NaN, ALL three are zeroed for consistency.
   335→func SanitizeStats(avg, min, max *schema.Float) {
   336→	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
   337→		*avg = schema.Float(0)
   338→		*min = schema.Float(0)
   339→		*max = schema.Float(0)
   340→	}
   341→}
   342→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file contains shared scope transformation logic used by both the internal
// metric store (pkg/metricstore) and the external cc-metric-store client
// (internal/metricstoreclient). It extracts the common algorithm for mapping
// between native metric scopes and requested scopes based on cluster topology.
package metricstore

import (
	"strconv"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Pre-converted scope strings avoid repeated string(MetricScope) allocations
// during query construction. Used in ScopeQueryResult.Type field.
var (
	HWThreadString     = string(schema.MetricScopeHWThread)
	CoreString         = string(schema.MetricScopeCore)
	MemoryDomainString = string(schema.MetricScopeMemoryDomain)
	SocketString       = string(schema.MetricScopeSocket)
	AcceleratorString  = string(schema.MetricScopeAccelerator)
)

// ScopeQueryResult is a package-independent intermediate type returned by
// BuildScopeQueries. Each consumer converts it to their own APIQuery type
// (adding Resolution and any other package-specific fields).
type ScopeQueryResult struct {
	Type      *string
	Metric    string
	Hostname  string
	TypeIds   []string
	Scope     schema.MetricScope
	Aggregate bool
}

// BuildScopeQueries generates scope query results for a given scope transformation.
// It returns a slice of results and a boolean indicating success.
// An empty slice means an expected exception (skip this combination).
// ok=false means an unhandled case (caller should return an error).
func BuildScopeQueries(
	nativeScope, requestedScope schema.MetricScope,
	metric, hostname string,
	topology *schema.Topology,
	hwthreads []int,
	accelerators []string,
) ([]ScopeQueryResult, bool) {
	scope := nativeScope.Max(requestedScope)
	results := []ScopeQueryResult{}

	hwthreadsStr := IntToStringSlice(hwthreads)

	// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
	if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
		if scope != schema.MetricScopeAccelerator {
			// Expected Exception -> Return Empty Slice
			return results, true
		}

		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &AcceleratorString,
			TypeIds:   accelerators,
			Scope:     schema.MetricScopeAccelerator,
		})
		return results, true
	}

	// Accelerator -> Node
	if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
		if len(accelerators) == 0 {
			// Expected Exception -> Return Empty Slice
			return results, true
		}

		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &AcceleratorString,
			TypeIds:   accelerators,
			Scope:     scope,
		})
		return results, true
	}

	// HWThread -> HWThread
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &HWThreadString,
			TypeIds:   hwthreadsStr,
			Scope:     scope,
		})
		return results, true
	}

	// HWThread -> Core
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		for _, core := range cores {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &HWThreadString,
				TypeIds:   IntToStringSlice(topology.Core[core]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// HWThread -> Socket
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		for _, socket := range sockets {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &HWThreadString,
				TypeIds:   IntToStringSlice(topology.Socket[socket]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// HWThread -> Node
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &HWThreadString,
			TypeIds:   hwthreadsStr,
			Scope:     scope,
		})
		return results, true
	}

	// Core -> Core
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &CoreString,
			TypeIds:   IntToStringSlice(cores),
			Scope:     scope,
		})
		return results, true
	}

	// Core -> Socket
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromCores(hwthreads)
		for _, socket := range sockets {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &CoreString,
				TypeIds:   IntToStringSlice(topology.Socket[socket]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// Core -> Node
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &CoreString,
			TypeIds:   IntToStringSlice(cores),
			Scope:     scope,
		})
		return results, true
	}

	// MemoryDomain -> MemoryDomain
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &MemoryDomainString,
			TypeIds:   IntToStringSlice(memDomains),
			Scope:     scope,
		})
		return results, true
	}

	// MemoryDomain -> Socket
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains)
		if err != nil {
			cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err)
			// Rare Error Case -> Still Continue -> Return Empty Slice
			return results, true
		}

		// Create a query for each socket
		for _, domains := range socketToDomains {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &MemoryDomainString,
				TypeIds:   IntToStringSlice(domains),
				Scope:     scope,
			})
		}
		return results, true
	}

	// MemoryDomain -> Node
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &MemoryDomainString,
			TypeIds:   IntToStringSlice(memDomains),
			Scope:     scope,
		})
		return results, true
	}

	// Socket -> Socket
	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &SocketString,
			TypeIds:   IntToStringSlice(sockets),
			Scope:     scope,
		})
		return results, true
	}

	// Socket -> Node
	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &SocketString,
			TypeIds:   IntToStringSlice(sockets),
			Scope:     scope,
		})
		return results, true
	}

	// Node -> Node
	if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
		results = append(results, ScopeQueryResult{
			Metric:   metric,
			Hostname: hostname,
			Scope:    scope,
		})
		return results, true
	}

	// Unhandled Case
	return nil, false
}

// IntToStringSlice converts a slice of integers to a slice of strings.
// Used to convert hardware thread/core/socket IDs from topology (int) to query TypeIds (string).
// Optimized to reuse a byte buffer for string conversion, reducing allocations.
func IntToStringSlice(is []int) []string {
	if len(is) == 0 {
		return nil
	}

	ss := make([]string, len(is))
	buf := make([]byte, 0, 16) // Reusable buffer for integer conversion
	for i, x := range is {
		buf = strconv.AppendInt(buf[:0], int64(x), 10)
		ss[i] = string(buf)
	}
	return ss
}

// ExtractTypeID returns the type ID at the given index from a query's TypeIds slice.
// Returns nil if queryType is nil (no type filtering). Logs a warning and returns nil
// if the index is out of range.
func ExtractTypeID(queryType *string, typeIds []string, ndx int, metric, hostname string) *string {
	if queryType == nil {
		return nil
	}
	if ndx < len(typeIds) {
		id := typeIds[ndx]
		return &id
	}
	cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
		ndx, len(typeIds), metric, hostname)
	return nil
}

// IsMetricRemovedForSubCluster checks whether a metric is marked as removed
// for the given subcluster in its per-subcluster configuration.
func IsMetricRemovedForSubCluster(mc *schema.MetricConfig, subCluster string) bool {
	for _, scConfig := range mc.SubClusters {
		if scConfig.Name == subCluster && scConfig.Remove {
			return true
		}
	}
	return false
}

// SanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
// If ANY of avg/min/max is NaN, ALL three are zeroed for consistency.
func SanitizeStats(avg, min, max *schema.Float) {
	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
		*avg = schema.Float(0)
		*min = schema.Float(0)
		*max = schema.Float(0)
	}
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"errors"
    10→	"math"
    11→
    12→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    13→	"github.com/ClusterCockpit/cc-lib/v2/util"
    14→)
    15→
    16→type Stats struct {
    17→	Samples int
    18→	Avg     schema.Float
    19→	Min     schema.Float
    20→	Max     schema.Float
    21→}
    22→
    23→func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
    24→	if from < b.start {
    25→		if b.prev != nil {
    26→			return b.prev.stats(from, to)
    27→		}
    28→		from = b.start
    29→	}
    30→
    31→	// TODO: Check if b.closed and if so and the full buffer is queried,
    32→	// use b.statistics instead of iterating over the buffer.
    33→
    34→	samples := 0
    35→	sum, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
    36→
    37→	var t int64
    38→	for t = from; t < to; t += b.frequency {
    39→		idx := int((t - b.start) / b.frequency)
    40→		if idx >= cap(b.data) {
    41→			b = b.next
    42→			if b == nil {
    43→				break
    44→			}
    45→			idx = int((t - b.start) / b.frequency)
    46→		}
    47→
    48→		if t < b.start || idx >= len(b.data) {
    49→			continue
    50→		}
    51→
    52→		xf := float64(b.data[idx])
    53→		if math.IsNaN(xf) {
    54→			continue
    55→		}
    56→
    57→		samples += 1
    58→		sum += xf
    59→		min = math.Min(min, xf)
    60→		max = math.Max(max, xf)
    61→	}
    62→
    63→	return Stats{
    64→		Samples: samples,
    65→		Avg:     schema.Float(sum) / schema.Float(samples),
    66→		Min:     schema.Float(min),
    67→		Max:     schema.Float(max),
    68→	}, from, t, nil
    69→}
    70→
    71→// Returns statistics for the requested metric on the selected node/level.
    72→// Data is aggregated to the selected level the same way as in `MemoryStore.Read`.
    73→// If `Stats.Samples` is zero, the statistics should not be considered as valid.
    74→func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int64) (*Stats, int64, int64, error) {
    75→	if from > to {
    76→		return nil, 0, 0, errors.New("invalid time range")
    77→	}
    78→
    79→	minfo, ok := m.Metrics[metric]
    80→	if !ok {
    81→		return nil, 0, 0, errors.New("unknown metric: " + metric)
    82→	}
    83→
    84→	n, samples := 0, 0
    85→	avg, min, max := schema.Float(0), math.MaxFloat32, -math.MaxFloat32
    86→	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
    87→		stats, cfrom, cto, err := b.stats(from, to)
    88→		if err != nil {
    89→			return err
    90→		}
    91→
    92→		if n == 0 {
    93→			from, to = cfrom, cto
    94→		} else if from != cfrom {
    95→			return ErrDataDoesNotAlignMissingFront
    96→		} else if to != cto {
    97→			return ErrDataDoesNotAlignMissingBack
    98→		}
    99→
   100→		samples += stats.Samples
   101→		avg += stats.Avg
   102→		min = math.Min(min, float64(stats.Min))
   103→		max = math.Max(max, float64(stats.Max))
   104→		n += 1
   105→		return nil
   106→	})
   107→	if err != nil {
   108→		return nil, 0, 0, err
   109→	}
   110→
   111→	if n == 0 {
   112→		return nil, 0, 0, ErrNoData
   113→	}
   114→
   115→	if minfo.Aggregation == AvgAggregation {
   116→		avg /= schema.Float(n)
   117→	} else if n > 1 && minfo.Aggregation != SumAggregation {
   118→		return nil, 0, 0, errors.New("invalid aggregation")
   119→	}
   120→
   121→	return &Stats{
   122→		Samples: samples,
   123→		Avg:     avg,
   124→		Min:     schema.Float(min),
   125→		Max:     schema.Float(max),
   126→	}, from, to, nil
   127→}
   128→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"errors"
	"math"

	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-lib/v2/util"
)

type Stats struct {
	Samples int
	Avg     schema.Float
	Min     schema.Float
	Max     schema.Float
}

func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
	if from < b.start {
		if b.prev != nil {
			return b.prev.stats(from, to)
		}
		from = b.start
	}

	// TODO: Check if b.closed and if so and the full buffer is queried,
	// use b.statistics instead of iterating over the buffer.

	samples := 0
	sum, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32

	var t int64
	for t = from; t < to; t += b.frequency {
		idx := int((t - b.start) / b.frequency)
		if idx >= cap(b.data) {
			b = b.next
			if b == nil {
				break
			}
			idx = int((t - b.start) / b.frequency)
		}

		if t < b.start || idx >= len(b.data) {
			continue
		}

		xf := float64(b.data[idx])
		if math.IsNaN(xf) {
			continue
		}

		samples += 1
		sum += xf
		min = math.Min(min, xf)
		max = math.Max(max, xf)
	}

	return Stats{
		Samples: samples,
		Avg:     schema.Float(sum) / schema.Float(samples),
		Min:     schema.Float(min),
		Max:     schema.Float(max),
	}, from, t, nil
}

// Returns statistics for the requested metric on the selected node/level.
// Data is aggregated to the selected level the same way as in `MemoryStore.Read`.
// If `Stats.Samples` is zero, the statistics should not be considered as valid.
func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int64) (*Stats, int64, int64, error) {
	if from > to {
		return nil, 0, 0, errors.New("invalid time range")
	}

	minfo, ok := m.Metrics[metric]
	if !ok {
		return nil, 0, 0, errors.New("unknown metric: " + metric)
	}

	n, samples := 0, 0
	avg, min, max := schema.Float(0), math.MaxFloat32, -math.MaxFloat32
	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
		stats, cfrom, cto, err := b.stats(from, to)
		if err != nil {
			return err
		}

		if n == 0 {
			from, to = cfrom, cto
		} else if from != cfrom {
			return ErrDataDoesNotAlignMissingFront
		} else if to != cto {
			return ErrDataDoesNotAlignMissingBack
		}

		samples += stats.Samples
		avg += stats.Avg
		min = math.Min(min, float64(stats.Min))
		max = math.Max(max, float64(stats.Max))
		n += 1
		return nil
	})
	if err != nil {
		return nil, 0, 0, err
	}

	if n == 0 {
		return nil, 0, 0, ErrNoData
	}

	if minfo.Aggregation == AvgAggregation {
		avg /= schema.Float(n)
	} else if n > 1 && minfo.Aggregation != SumAggregation {
		return nil, 0, 0, errors.New("invalid aggregation")
	}

	return &Stats{
		Samples: samples,
		Avg:     avg,
		Min:     schema.Float(min),
		Max:     schema.Float(max),
	}, from, to, nil
}