     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file contains the API types and data fetching logic for querying metric data
     7→// from the in-memory metric store. It provides structures for building complex queries
     8→// with support for aggregation, scaling, padding, and statistics computation.
     9→package metricstore
    10→
    11→import (
    12→	"errors"
    13→	"fmt"
    14→	"math"
    15→
    16→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    17→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    18→	"github.com/ClusterCockpit/cc-lib/v2/util"
    19→)
    20→
    21→var (
    22→	// ErrNoHostOrMetric is returned when the metric store does not find the host or the metric
    23→	ErrNoHostOrMetric error = errors.New("[METRICSTORE]> metric or host not found")
    24→	// ErrInvalidTimeRange is returned when a query has 'from' >= 'to'
    25→	ErrInvalidTimeRange = errors.New("[METRICSTORE]> invalid time range: 'from' must be before 'to'")
    26→	// ErrEmptyCluster is returned when a query with ForAllNodes has no cluster specified
    27→	ErrEmptyCluster = errors.New("[METRICSTORE]> cluster name cannot be empty")
    28→)
    29→
    30→// APIMetricData represents the response data for a single metric query.
    31→//
    32→// It contains both the time-series data points and computed statistics (avg, min, max).
    33→// If an error occurred during data retrieval, the Error field will be set and other
    34→// fields may be incomplete.
    35→type APIMetricData struct {
    36→	Error      *string           `json:"error,omitempty"`
    37→	Data       schema.FloatArray `json:"data,omitempty"`
    38→	From       int64             `json:"from"`
    39→	To         int64             `json:"to"`
    40→	Resolution int64             `json:"resolution"`
    41→	Avg        schema.Float      `json:"avg"`
    42→	Min        schema.Float      `json:"min"`
    43→	Max        schema.Float      `json:"max"`
    44→}
    45→
    46→// APIQueryRequest represents a batch query request for metric data.
    47→//
    48→// It supports two modes of operation:
    49→//  1. Explicit queries via the Queries field
    50→//  2. Automatic query generation via ForAllNodes (queries all specified metrics for all nodes in the cluster)
    51→//
    52→// The request can be customized with flags to include/exclude statistics, raw data, and padding.
    53→type APIQueryRequest struct {
    54→	Cluster     string     `json:"cluster"`
    55→	Queries     []APIQuery `json:"queries"`
    56→	ForAllNodes []string   `json:"for-all-nodes"`
    57→	From        int64      `json:"from"`
    58→	To          int64      `json:"to"`
    59→	WithStats   bool       `json:"with-stats"`
    60→	WithData    bool       `json:"with-data"`
    61→	WithPadding bool       `json:"with-padding"`
    62→}
    63→
    64→// APIQueryResponse represents the response to an APIQueryRequest.
    65→//
    66→// Results is a 2D array where each outer element corresponds to a query,
    67→// and each inner element corresponds to a selector within that query
    68→// (e.g., multiple CPUs or cores).
    69→type APIQueryResponse struct {
    70→	Queries []APIQuery        `json:"queries,omitempty"`
    71→	Results [][]APIMetricData `json:"results"`
    72→}
    73→
    74→// APIQuery represents a single metric query with optional hierarchical selectors.
    75→//
    76→// The hierarchical selection works as follows:
    77→//   - Hostname: The node to query
    78→//   - Type + TypeIds: First level of hierarchy (e.g., "cpu" + ["0", "1", "2"])
    79→//   - SubType + SubTypeIds: Second level of hierarchy (e.g., "core" + ["0", "1"])
    80→//
    81→// If Aggregate is true, data from multiple type/subtype IDs will be aggregated according
    82→// to the metric's aggregation strategy. Otherwise, separate results are returned for each combination.
    83→type APIQuery struct {
    84→	Type        *string      `json:"type,omitempty"`
    85→	SubType     *string      `json:"subtype,omitempty"`
    86→	Metric      string       `json:"metric"`
    87→	Hostname    string       `json:"host"`
    88→	Resolution  int64        `json:"resolution"`
    89→	TypeIds     []string     `json:"type-ids,omitempty"`
    90→	SubTypeIds  []string     `json:"subtype-ids,omitempty"`
    91→	ScaleFactor schema.Float `json:"scale-by,omitempty"`
    92→	Aggregate   bool         `json:"aggreg"`
    93→}
    94→
    95→// AddStats computes and populates the Avg, Min, and Max fields from the Data array.
    96→//
    97→// NaN values in the data are ignored during computation. If all values are NaN,
    98→// the statistics fields will be set to NaN.
    99→//
   100→// TODO: Optimize this, just like the stats endpoint!
   101→func (data *APIMetricData) AddStats() {
   102→	n := 0
   103→	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
   104→	for _, x := range data.Data {
   105→		if x.IsNaN() {
   106→			continue
   107→		}
   108→
   109→		n += 1
   110→		sum += float64(x)
   111→		min = math.Min(min, float64(x))
   112→		max = math.Max(max, float64(x))
   113→	}
   114→
   115→	if n > 0 {
   116→		avg := sum / float64(n)
   117→		data.Avg = schema.Float(avg)
   118→		data.Min = schema.Float(min)
   119→		data.Max = schema.Float(max)
   120→	} else {
   121→		data.Avg, data.Min, data.Max = schema.NaN, schema.NaN, schema.NaN
   122→	}
   123→}
   124→
   125→// ScaleBy multiplies all data points and statistics by the given factor.
   126→//
   127→// This is commonly used for unit conversion (e.g., bytes to gigabytes).
   128→// Scaling by 0 or 1 is a no-op for performance reasons.
   129→func (data *APIMetricData) ScaleBy(f schema.Float) {
   130→	if f == 0 || f == 1 {
   131→		return
   132→	}
   133→
   134→	data.Avg *= f
   135→	data.Min *= f
   136→	data.Max *= f
   137→	for i := 0; i < len(data.Data); i++ {
   138→		data.Data[i] *= f
   139→	}
   140→}
   141→
   142→// PadDataWithNull pads the beginning of the data array with NaN values if needed.
   143→//
   144→// This ensures that the data aligns with the requested 'from' timestamp, even if
   145→// the metric store doesn't have data for the earliest time points. This is useful
   146→// for maintaining consistent array indexing across multiple queries.
   147→//
   148→// Parameters:
   149→//   - ms: MemoryStore instance to lookup metric configuration
   150→//   - from: The requested start timestamp
   151→//   - to: The requested end timestamp (unused but kept for API consistency)
   152→//   - metric: The metric name to lookup frequency information
   153→func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metric string) {
   154→	minfo, ok := ms.Metrics[metric]
   155→	if !ok {
   156→		return
   157→	}
   158→
   159→	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
   160→		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
   161→		ndata := make([]schema.Float, 0, padfront+len(data.Data))
   162→		for range padfront {
   163→			ndata = append(ndata, schema.NaN)
   164→		}
   165→		for j := 0; j < len(data.Data); j++ {
   166→			ndata = append(ndata, data.Data[j])
   167→		}
   168→		data.Data = ndata
   169→	}
   170→}
   171→
   172→// FetchData executes a batch metric query request and returns the results.
   173→//
   174→// This is the primary API for retrieving metric data from the memory store. It supports:
   175→//   - Individual queries via req.Queries
   176→//   - Batch queries for all nodes via req.ForAllNodes
   177→//   - Hierarchical selector construction (cluster → host → type → subtype)
   178→//   - Optional statistics computation (avg, min, max)
   179→//   - Optional data scaling
   180→//   - Optional data padding with NaN values
   181→//
   182→// The function constructs selectors based on the query parameters and calls MemoryStore.Read()
   183→// for each selector. If a query specifies Aggregate=false with multiple type/subtype IDs,
   184→// separate results are returned for each combination.
   185→//
   186→// Parameters:
   187→//   - req: The query request containing queries, time range, and options
   188→//
   189→// Returns:
   190→//   - APIQueryResponse containing results for each query, or error if validation fails
   191→//
   192→// Errors:
   193→//   - ErrInvalidTimeRange if req.From > req.To
   194→//   - ErrEmptyCluster if req.ForAllNodes is used without specifying a cluster
   195→//   - Error if MemoryStore is not initialized
   196→//   - Individual query errors are stored in APIMetricData.Error field
   197→func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
   198→	if req.From > req.To {
   199→		return nil, ErrInvalidTimeRange
   200→	}
   201→	if req.Cluster == "" && req.ForAllNodes != nil {
   202→		return nil, ErrEmptyCluster
   203→	}
   204→
   205→	req.WithData = true
   206→	ms := GetMemoryStore()
   207→	if ms == nil {
   208→		return nil, fmt.Errorf("[METRICSTORE]> memorystore not initialized")
   209→	}
   210→
   211→	response := APIQueryResponse{
   212→		Results: make([][]APIMetricData, 0, len(req.Queries)),
   213→	}
   214→	if req.ForAllNodes != nil {
   215→		nodes := ms.ListChildren([]string{req.Cluster})
   216→		for _, node := range nodes {
   217→			for _, metric := range req.ForAllNodes {
   218→				q := APIQuery{
   219→					Metric:   metric,
   220→					Hostname: node,
   221→				}
   222→				req.Queries = append(req.Queries, q)
   223→				response.Queries = append(response.Queries, q)
   224→			}
   225→		}
   226→	}
   227→
   228→	for _, query := range req.Queries {
   229→		sels := make([]util.Selector, 0, 1)
   230→		if query.Aggregate || query.Type == nil {
   231→			sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}}
   232→			if query.Type != nil {
   233→				if len(query.TypeIds) == 1 {
   234→					sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]})
   235→				} else {
   236→					ids := make([]string, len(query.TypeIds))
   237→					for i, id := range query.TypeIds {
   238→						ids[i] = *query.Type + id
   239→					}
   240→					sel = append(sel, util.SelectorElement{Group: ids})
   241→				}
   242→
   243→				if query.SubType != nil {
   244→					if len(query.SubTypeIds) == 1 {
   245→						sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
   246→					} else {
   247→						ids := make([]string, len(query.SubTypeIds))
   248→						for i, id := range query.SubTypeIds {
   249→							ids[i] = *query.SubType + id
   250→						}
   251→						sel = append(sel, util.SelectorElement{Group: ids})
   252→					}
   253→				}
   254→			}
   255→			sels = append(sels, sel)
   256→		} else {
   257→			for _, typeID := range query.TypeIds {
   258→				if query.SubType != nil {
   259→					for _, subTypeID := range query.SubTypeIds {
   260→						sels = append(sels, util.Selector{
   261→							{String: req.Cluster},
   262→							{String: query.Hostname},
   263→							{String: *query.Type + typeID},
   264→							{String: *query.SubType + subTypeID},
   265→						})
   266→					}
   267→				} else {
   268→					sels = append(sels, util.Selector{
   269→						{String: req.Cluster},
   270→						{String: query.Hostname},
   271→						{String: *query.Type + typeID},
   272→					})
   273→				}
   274→			}
   275→		}
   276→
   277→		var err error
   278→		res := make([]APIMetricData, 0, len(sels))
   279→		for _, sel := range sels {
   280→			data := APIMetricData{}
   281→
   282→			data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
   283→			if err != nil {
   284→				// Skip Error If Just Missing Host or Metric, Continue
   285→				// Empty Return For Metric Handled Gracefully By Frontend
   286→				if err != ErrNoHostOrMetric {
   287→					msg := err.Error()
   288→					data.Error = &msg
   289→					res = append(res, data)
   290→				} else {
   291→					cclog.Warnf("failed to fetch '%s' from host '%s' (cluster: %s): %s", query.Metric, query.Hostname, req.Cluster, err.Error())
   292→				}
   293→				continue
   294→			}
   295→
   296→			if req.WithStats {
   297→				data.AddStats()
   298→			}
   299→			if query.ScaleFactor != 0 {
   300→				data.ScaleBy(query.ScaleFactor)
   301→			}
   302→			if req.WithPadding {
   303→				data.PadDataWithNull(ms, req.From, req.To, query.Metric)
   304→			}
   305→			if !req.WithData {
   306→				data.Data = nil
   307→			}
   308→			res = append(res, data)
   309→		}
   310→		response.Results = append(response.Results, res)
   311→	}
   312→
   313→	return &response, nil
   314→}
   315→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file contains the API types and data fetching logic for querying metric data
// from the in-memory metric store. It provides structures for building complex queries
// with support for aggregation, scaling, padding, and statistics computation.
package metricstore

import (
	"errors"
	"fmt"
	"math"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-lib/v2/util"
)

var (
	// ErrNoHostOrMetric is returned when the metric store does not find the host or the metric
	ErrNoHostOrMetric error = errors.New("[METRICSTORE]> metric or host not found")
	// ErrInvalidTimeRange is returned when a query has 'from' >= 'to'
	ErrInvalidTimeRange = errors.New("[METRICSTORE]> invalid time range: 'from' must be before 'to'")
	// ErrEmptyCluster is returned when a query with ForAllNodes has no cluster specified
	ErrEmptyCluster = errors.New("[METRICSTORE]> cluster name cannot be empty")
)

// APIMetricData represents the response data for a single metric query.
//
// It contains both the time-series data points and computed statistics (avg, min, max).
// If an error occurred during data retrieval, the Error field will be set and other
// fields may be incomplete.
type APIMetricData struct {
	Error      *string           `json:"error,omitempty"`
	Data       schema.FloatArray `json:"data,omitempty"`
	From       int64             `json:"from"`
	To         int64             `json:"to"`
	Resolution int64             `json:"resolution"`
	Avg        schema.Float      `json:"avg"`
	Min        schema.Float      `json:"min"`
	Max        schema.Float      `json:"max"`
}

// APIQueryRequest represents a batch query request for metric data.
//
// It supports two modes of operation:
//  1. Explicit queries via the Queries field
//  2. Automatic query generation via ForAllNodes (queries all specified metrics for all nodes in the cluster)
//
// The request can be customized with flags to include/exclude statistics, raw data, and padding.
type APIQueryRequest struct {
	Cluster     string     `json:"cluster"`
	Queries     []APIQuery `json:"queries"`
	ForAllNodes []string   `json:"for-all-nodes"`
	From        int64      `json:"from"`
	To          int64      `json:"to"`
	WithStats   bool       `json:"with-stats"`
	WithData    bool       `json:"with-data"`
	WithPadding bool       `json:"with-padding"`
}

// APIQueryResponse represents the response to an APIQueryRequest.
//
// Results is a 2D array where each outer element corresponds to a query,
// and each inner element corresponds to a selector within that query
// (e.g., multiple CPUs or cores).
type APIQueryResponse struct {
	Queries []APIQuery        `json:"queries,omitempty"`
	Results [][]APIMetricData `json:"results"`
}

// APIQuery represents a single metric query with optional hierarchical selectors.
//
// The hierarchical selection works as follows:
//   - Hostname: The node to query
//   - Type + TypeIds: First level of hierarchy (e.g., "cpu" + ["0", "1", "2"])
//   - SubType + SubTypeIds: Second level of hierarchy (e.g., "core" + ["0", "1"])
//
// If Aggregate is true, data from multiple type/subtype IDs will be aggregated according
// to the metric's aggregation strategy. Otherwise, separate results are returned for each combination.
type APIQuery struct {
	Type        *string      `json:"type,omitempty"`
	SubType     *string      `json:"subtype,omitempty"`
	Metric      string       `json:"metric"`
	Hostname    string       `json:"host"`
	Resolution  int64        `json:"resolution"`
	TypeIds     []string     `json:"type-ids,omitempty"`
	SubTypeIds  []string     `json:"subtype-ids,omitempty"`
	ScaleFactor schema.Float `json:"scale-by,omitempty"`
	Aggregate   bool         `json:"aggreg"`
}

// AddStats computes and populates the Avg, Min, and Max fields from the Data array.
//
// NaN values in the data are ignored during computation. If all values are NaN,
// the statistics fields will be set to NaN.
//
// TODO: Optimize this, just like the stats endpoint!
func (data *APIMetricData) AddStats() {
	n := 0
	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
	for _, x := range data.Data {
		if x.IsNaN() {
			continue
		}

		n += 1
		sum += float64(x)
		min = math.Min(min, float64(x))
		max = math.Max(max, float64(x))
	}

	if n > 0 {
		avg := sum / float64(n)
		data.Avg = schema.Float(avg)
		data.Min = schema.Float(min)
		data.Max = schema.Float(max)
	} else {
		data.Avg, data.Min, data.Max = schema.NaN, schema.NaN, schema.NaN
	}
}

// ScaleBy multiplies all data points and statistics by the given factor.
//
// This is commonly used for unit conversion (e.g., bytes to gigabytes).
// Scaling by 0 or 1 is a no-op for performance reasons.
func (data *APIMetricData) ScaleBy(f schema.Float) {
	if f == 0 || f == 1 {
		return
	}

	data.Avg *= f
	data.Min *= f
	data.Max *= f
	for i := 0; i < len(data.Data); i++ {
		data.Data[i] *= f
	}
}

// PadDataWithNull pads the beginning of the data array with NaN values if needed.
//
// This ensures that the data aligns with the requested 'from' timestamp, even if
// the metric store doesn't have data for the earliest time points. This is useful
// for maintaining consistent array indexing across multiple queries.
//
// Parameters:
//   - ms: MemoryStore instance to lookup metric configuration
//   - from: The requested start timestamp
//   - to: The requested end timestamp (unused but kept for API consistency)
//   - metric: The metric name to lookup frequency information
func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metric string) {
	minfo, ok := ms.Metrics[metric]
	if !ok {
		return
	}

	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
		ndata := make([]schema.Float, 0, padfront+len(data.Data))
		for range padfront {
			ndata = append(ndata, schema.NaN)
		}
		for j := 0; j < len(data.Data); j++ {
			ndata = append(ndata, data.Data[j])
		}
		data.Data = ndata
	}
}

// FetchData executes a batch metric query request and returns the results.
//
// This is the primary API for retrieving metric data from the memory store. It supports:
//   - Individual queries via req.Queries
//   - Batch queries for all nodes via req.ForAllNodes
//   - Hierarchical selector construction (cluster → host → type → subtype)
//   - Optional statistics computation (avg, min, max)
//   - Optional data scaling
//   - Optional data padding with NaN values
//
// The function constructs selectors based on the query parameters and calls MemoryStore.Read()
// for each selector. If a query specifies Aggregate=false with multiple type/subtype IDs,
// separate results are returned for each combination.
//
// Parameters:
//   - req: The query request containing queries, time range, and options
//
// Returns:
//   - APIQueryResponse containing results for each query, or error if validation fails
//
// Errors:
//   - ErrInvalidTimeRange if req.From > req.To
//   - ErrEmptyCluster if req.ForAllNodes is used without specifying a cluster
//   - Error if MemoryStore is not initialized
//   - Individual query errors are stored in APIMetricData.Error field
func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
	if req.From > req.To {
		return nil, ErrInvalidTimeRange
	}
	if req.Cluster == "" && req.ForAllNodes != nil {
		return nil, ErrEmptyCluster
	}

	req.WithData = true
	ms := GetMemoryStore()
	if ms == nil {
		return nil, fmt.Errorf("[METRICSTORE]> memorystore not initialized")
	}

	response := APIQueryResponse{
		Results: make([][]APIMetricData, 0, len(req.Queries)),
	}
	if req.ForAllNodes != nil {
		nodes := ms.ListChildren([]string{req.Cluster})
		for _, node := range nodes {
			for _, metric := range req.ForAllNodes {
				q := APIQuery{
					Metric:   metric,
					Hostname: node,
				}
				req.Queries = append(req.Queries, q)
				response.Queries = append(response.Queries, q)
			}
		}
	}

	for _, query := range req.Queries {
		sels := make([]util.Selector, 0, 1)
		if query.Aggregate || query.Type == nil {
			sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}}
			if query.Type != nil {
				if len(query.TypeIds) == 1 {
					sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]})
				} else {
					ids := make([]string, len(query.TypeIds))
					for i, id := range query.TypeIds {
						ids[i] = *query.Type + id
					}
					sel = append(sel, util.SelectorElement{Group: ids})
				}

				if query.SubType != nil {
					if len(query.SubTypeIds) == 1 {
						sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
					} else {
						ids := make([]string, len(query.SubTypeIds))
						for i, id := range query.SubTypeIds {
							ids[i] = *query.SubType + id
						}
						sel = append(sel, util.SelectorElement{Group: ids})
					}
				}
			}
			sels = append(sels, sel)
		} else {
			for _, typeID := range query.TypeIds {
				if query.SubType != nil {
					for _, subTypeID := range query.SubTypeIds {
						sels = append(sels, util.Selector{
							{String: req.Cluster},
							{String: query.Hostname},
							{String: *query.Type + typeID},
							{String: *query.SubType + subTypeID},
						})
					}
				} else {
					sels = append(sels, util.Selector{
						{String: req.Cluster},
						{String: query.Hostname},
						{String: *query.Type + typeID},
					})
				}
			}
		}

		var err error
		res := make([]APIMetricData, 0, len(sels))
		for _, sel := range sels {
			data := APIMetricData{}

			data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
			if err != nil {
				// Skip Error If Just Missing Host or Metric, Continue
				// Empty Return For Metric Handled Gracefully By Frontend
				if err != ErrNoHostOrMetric {
					msg := err.Error()
					data.Error = &msg
					res = append(res, data)
				} else {
					cclog.Warnf("failed to fetch '%s' from host '%s' (cluster: %s): %s", query.Metric, query.Hostname, req.Cluster, err.Error())
				}
				continue
			}

			if req.WithStats {
				data.AddStats()
			}
			if query.ScaleFactor != 0 {
				data.ScaleBy(query.ScaleFactor)
			}
			if req.WithPadding {
				data.PadDataWithNull(ms, req.From, req.To, query.Metric)
			}
			if !req.WithData {
				data.Data = nil
			}
			res = append(res, data)
		}
		response.Results = append(response.Results, res)
	}

	return &response, nil
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"encoding/json"
    10→	"fmt"
    11→	"time"
    12→
    13→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    14→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    15→)
    16→
    17→// HealthCheckResponse represents the result of a health check operation.
    18→type HealthCheckResponse struct {
    19→	Status schema.MonitoringState
    20→	Error  error
    21→}
    22→
    23→// HealthCheckResult holds the monitoring state and raw JSON health metrics
    24→// for a single node as determined by HealthCheck.
    25→type HealthCheckResult struct {
    26→	State         schema.MonitoringState
    27→	HealthMetrics string // JSON: {"missing":[...],"degraded":[...]}
    28→}
    29→
    30→// MaxMissingDataPoints is the threshold for stale data detection.
    31→// A buffer is considered healthy if the gap between its last data point
    32→// and the current time is within MaxMissingDataPoints * frequency.
    33→const MaxMissingDataPoints int64 = 5
    34→
    35→// bufferExists returns true if the buffer is non-nil and contains data.
    36→func (b *buffer) bufferExists() bool {
    37→	if b == nil || b.data == nil || len(b.data) == 0 {
    38→		return false
    39→	}
    40→
    41→	return true
    42→}
    43→
    44→// isBufferHealthy returns true if the buffer has recent data within
    45→// MaxMissingDataPoints * frequency of the current time.
    46→func (b *buffer) isBufferHealthy() bool {
    47→	bufferEnd := b.start + b.frequency*int64(len(b.data))
    48→	t := time.Now().Unix()
    49→
    50→	return t-bufferEnd <= MaxMissingDataPoints*b.frequency
    51→}
    52→
    53→// collectMetricStatus walks the subtree rooted at l and classifies each
    54→// expected metric into the healthy or degraded map.
    55→//
    56→// Classification rules (evaluated per buffer, pessimistic):
    57→//   - A single stale buffer marks the metric as degraded permanently.
    58→//   - A healthy buffer only counts if no stale buffer has been seen.
    59→//   - Metrics absent from the global config or without any buffer remain
    60→//     in neither map and are later reported as missing.
    61→func (l *Level) collectMetricStatus(m *MemoryStore, expectedMetrics []string, healthy, degraded map[string]bool) {
    62→	l.lock.RLock()
    63→	defer l.lock.RUnlock()
    64→
    65→	for _, metricName := range expectedMetrics {
    66→		if degraded[metricName] {
    67→			continue // already degraded, cannot improve
    68→		}
    69→		mc := m.Metrics[metricName]
    70→		b := l.metrics[mc.offset]
    71→		if b.bufferExists() {
    72→			if !b.isBufferHealthy() {
    73→				degraded[metricName] = true
    74→				delete(healthy, metricName)
    75→			} else if !degraded[metricName] {
    76→				healthy[metricName] = true
    77→			}
    78→		}
    79→	}
    80→
    81→	for _, lvl := range l.children {
    82→		lvl.collectMetricStatus(m, expectedMetrics, healthy, degraded)
    83→	}
    84→}
    85→
    86→// getHealthyMetrics walks the complete subtree rooted at l and classifies
    87→// each expected metric by comparing the collected status against the
    88→// expected list.
    89→//
    90→// Returns:
    91→//   - missingList: metrics not found in global config or without any buffer
    92→//   - degradedList: metrics with at least one stale buffer in the subtree
    93→func (l *Level) getHealthyMetrics(m *MemoryStore, expectedMetrics []string) ([]string, []string) {
    94→	healthy := make(map[string]bool, len(expectedMetrics))
    95→	degraded := make(map[string]bool)
    96→
    97→	l.collectMetricStatus(m, expectedMetrics, healthy, degraded)
    98→
    99→	missingList := make([]string, 0)
   100→	degradedList := make([]string, 0)
   101→
   102→	for _, metricName := range expectedMetrics {
   103→		if healthy[metricName] {
   104→			continue
   105→		}
   106→
   107→		if degraded[metricName] {
   108→			degradedList = append(degradedList, metricName)
   109→		} else {
   110→			missingList = append(missingList, metricName)
   111→		}
   112→	}
   113→
   114→	return degradedList, missingList
   115→}
   116→
   117→// GetHealthyMetrics returns missing and degraded metric lists for a node.
   118→//
   119→// It walks the metric tree starting from the node identified by selector
   120→// and classifies each expected metric:
   121→//   - Missing: no buffer anywhere in the subtree, or metric not in global config
   122→//   - Degraded: at least one stale buffer exists in the subtree
   123→//
   124→// Metrics present in expectedMetrics but absent from both returned lists
   125→// are considered fully healthy.
   126→func (m *MemoryStore) GetHealthyMetrics(selector []string, expectedMetrics []string) ([]string, []string, error) {
   127→	lvl := m.root.findLevel(selector)
   128→	if lvl == nil {
   129→		return nil, nil, fmt.Errorf("[METRICSTORE]> GetHealthyMetrics: host not found: %#v", selector)
   130→	}
   131→
   132→	degradedList, missingList := lvl.getHealthyMetrics(m, expectedMetrics)
   133→	return degradedList, missingList, nil
   134→}
   135→
   136→type HealthCheckReq struct {
   137→	Cluster     string   `json:"cluster" example:"fritz"`
   138→	Nodes       []string `json:"nodes"`
   139→	MetricNames []string `json:"metric-names"`
   140→}
   141→
   142→// HealthCheck evaluates multiple nodes against a set of expected metrics
   143→// and returns a monitoring state per node.
   144→//
   145→// States:
   146→//   - MonitoringStateFull: all expected metrics are healthy
   147→//   - MonitoringStatePartial: some metrics are missing or degraded
   148→//   - MonitoringStateFailed: node not found, or no healthy metrics at all
   149→func (m *MemoryStore) HealthCheck(cluster string,
   150→	nodes []string, expectedMetrics []string,
   151→) (map[string]HealthCheckResult, error) {
   152→	results := make(map[string]HealthCheckResult, len(nodes))
   153→
   154→	for _, hostname := range nodes {
   155→		selector := []string{cluster, hostname}
   156→
   157→		degradedList, missingList, err := m.GetHealthyMetrics(selector, expectedMetrics)
   158→		if err != nil {
   159→			results[hostname] = HealthCheckResult{State: schema.MonitoringStateFailed}
   160→			continue
   161→		}
   162→
   163→		degradedCount := len(degradedList)
   164→		missingCount := len(missingList)
   165→
   166→		healthyCount := len(expectedMetrics) - degradedCount - missingCount
   167→
   168→		if degradedCount > 0 {
   169→			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "degraded metrics:", degradedList)
   170→		}
   171→		if missingCount > 0 {
   172→			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "missing metrics:", missingList)
   173→		}
   174→
   175→		var state schema.MonitoringState
   176→		switch {
   177→		case degradedCount == 0 && missingCount == 0:
   178→			state = schema.MonitoringStateFull
   179→		case healthyCount == 0:
   180→			state = schema.MonitoringStateFailed
   181→		default:
   182→			state = schema.MonitoringStatePartial
   183→		}
   184→
   185→		hm, _ := json.Marshal(map[string][]string{
   186→			"missing":  missingList,
   187→			"degraded": degradedList,
   188→		})
   189→
   190→		results[hostname] = HealthCheckResult{
   191→			State:         state,
   192→			HealthMetrics: string(hm),
   193→		}
   194→	}
   195→
   196→	return results, nil
   197→}
   198→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package metricstore

import (
	"encoding/json"
	"fmt"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// HealthCheckResponse represents the result of a health check operation.
type HealthCheckResponse struct {
	Status schema.MonitoringState
	Error  error
}

// HealthCheckResult holds the monitoring state and raw JSON health metrics
// for a single node as determined by HealthCheck.
type HealthCheckResult struct {
	State         schema.MonitoringState
	HealthMetrics string // JSON: {"missing":[...],"degraded":[...]}
}

// MaxMissingDataPoints is the threshold for stale data detection.
// A buffer is considered healthy if the gap between its last data point
// and the current time is within MaxMissingDataPoints * frequency.
const MaxMissingDataPoints int64 = 5

// bufferExists returns true if the buffer is non-nil and contains data.
func (b *buffer) bufferExists() bool {
	if b == nil || b.data == nil || len(b.data) == 0 {
		return false
	}

	return true
}

// isBufferHealthy returns true if the buffer has recent data within
// MaxMissingDataPoints * frequency of the current time.
func (b *buffer) isBufferHealthy() bool {
	bufferEnd := b.start + b.frequency*int64(len(b.data))
	t := time.Now().Unix()

	return t-bufferEnd <= MaxMissingDataPoints*b.frequency
}

// collectMetricStatus walks the subtree rooted at l and classifies each
// expected metric into the healthy or degraded map.
//
// Classification rules (evaluated per buffer, pessimistic):
//   - A single stale buffer marks the metric as degraded permanently.
//   - A healthy buffer only counts if no stale buffer has been seen.
//   - Metrics absent from the global config or without any buffer remain
//     in neither map and are later reported as missing.
func (l *Level) collectMetricStatus(m *MemoryStore, expectedMetrics []string, healthy, degraded map[string]bool) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	for _, metricName := range expectedMetrics {
		if degraded[metricName] {
			continue // already degraded, cannot improve
		}
		mc := m.Metrics[metricName]
		b := l.metrics[mc.offset]
		if b.bufferExists() {
			if !b.isBufferHealthy() {
				degraded[metricName] = true
				delete(healthy, metricName)
			} else if !degraded[metricName] {
				healthy[metricName] = true
			}
		}
	}

	for _, lvl := range l.children {
		lvl.collectMetricStatus(m, expectedMetrics, healthy, degraded)
	}
}

// getHealthyMetrics walks the complete subtree rooted at l and classifies
// each expected metric by comparing the collected status against the
// expected list.
//
// Returns:
//   - missingList: metrics not found in global config or without any buffer
//   - degradedList: metrics with at least one stale buffer in the subtree
func (l *Level) getHealthyMetrics(m *MemoryStore, expectedMetrics []string) ([]string, []string) {
	healthy := make(map[string]bool, len(expectedMetrics))
	degraded := make(map[string]bool)

	l.collectMetricStatus(m, expectedMetrics, healthy, degraded)

	missingList := make([]string, 0)
	degradedList := make([]string, 0)

	for _, metricName := range expectedMetrics {
		if healthy[metricName] {
			continue
		}

		if degraded[metricName] {
			degradedList = append(degradedList, metricName)
		} else {
			missingList = append(missingList, metricName)
		}
	}

	return degradedList, missingList
}

// GetHealthyMetrics returns missing and degraded metric lists for a node.
//
// It walks the metric tree starting from the node identified by selector
// and classifies each expected metric:
//   - Missing: no buffer anywhere in the subtree, or metric not in global config
//   - Degraded: at least one stale buffer exists in the subtree
//
// Metrics present in expectedMetrics but absent from both returned lists
// are considered fully healthy.
func (m *MemoryStore) GetHealthyMetrics(selector []string, expectedMetrics []string) ([]string, []string, error) {
	lvl := m.root.findLevel(selector)
	if lvl == nil {
		return nil, nil, fmt.Errorf("[METRICSTORE]> GetHealthyMetrics: host not found: %#v", selector)
	}

	degradedList, missingList := lvl.getHealthyMetrics(m, expectedMetrics)
	return degradedList, missingList, nil
}

type HealthCheckReq struct {
	Cluster     string   `json:"cluster" example:"fritz"`
	Nodes       []string `json:"nodes"`
	MetricNames []string `json:"metric-names"`
}

// HealthCheck evaluates multiple nodes against a set of expected metrics
// and returns a monitoring state per node.
//
// States:
//   - MonitoringStateFull: all expected metrics are healthy
//   - MonitoringStatePartial: some metrics are missing or degraded
//   - MonitoringStateFailed: node not found, or no healthy metrics at all
func (m *MemoryStore) HealthCheck(cluster string,
	nodes []string, expectedMetrics []string,
) (map[string]HealthCheckResult, error) {
	results := make(map[string]HealthCheckResult, len(nodes))

	for _, hostname := range nodes {
		selector := []string{cluster, hostname}

		degradedList, missingList, err := m.GetHealthyMetrics(selector, expectedMetrics)
		if err != nil {
			results[hostname] = HealthCheckResult{State: schema.MonitoringStateFailed}
			continue
		}

		degradedCount := len(degradedList)
		missingCount := len(missingList)

		healthyCount := len(expectedMetrics) - degradedCount - missingCount

		if degradedCount > 0 {
			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "degraded metrics:", degradedList)
		}
		if missingCount > 0 {
			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "missing metrics:", missingList)
		}

		var state schema.MonitoringState
		switch {
		case degradedCount == 0 && missingCount == 0:
			state = schema.MonitoringStateFull
		case healthyCount == 0:
			state = schema.MonitoringStateFailed
		default:
			state = schema.MonitoringStatePartial
		}

		hm, _ := json.Marshal(map[string][]string{
			"missing":  missingList,
			"degraded": degradedList,
		})

		results[hostname] = HealthCheckResult{
			State:         state,
			HealthMetrics: string(hm),
		}
	}

	return results, nil
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides config.go: Configuration structures and metric management.
     7→//
     8→// # Configuration Hierarchy
     9→//
    10→// The metricstore package uses nested configuration structures:
    11→//
    12→//	MetricStoreConfig (Keys)
    13→//	├─ NumWorkers: Parallel checkpoint/archive workers
    14→//	├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval)
    15→//	├─ MemoryCap: Memory limit in bytes (triggers forceFree)
    16→//	├─ Checkpoints: Persistence configuration
    17→//	│  ├─ FileFormat: "json" or "wal" (default: "wal")
    18→//	│  └─ RootDir: Checkpoint storage path
    19→//	├─ Cleanup: Long-term storage configuration (interval = RetentionInMemory)
    20→//	│  ├─ RootDir: Archive storage path (archive mode only)
    21→//	│  └─ Mode: "delete" or "archive"
    22→//	├─ Debug: Development/debugging options
    23→//	└─ Subscriptions: NATS topic subscriptions for metric ingestion
    24→//
    25→// # Metric Configuration
    26→//
    27→// Each metric (e.g., "cpu_load", "mem_used") has a MetricConfig entry in the global
    28→// Metrics map, defining:
    29→//
    30→//   - Frequency: Measurement interval in seconds
    31→//   - Aggregation: How to combine values (sum/avg/none) when transforming scopes
    32→//   - offset: Internal index into Level.metrics slice (assigned during Init)
    33→//
    34→// # AggregationStrategy
    35→//
    36→// Determines how to combine metric values when aggregating from finer to coarser scopes:
    37→//
    38→//   - NoAggregation: Do not combine (incompatible scopes)
    39→//   - SumAggregation: Add values (e.g., power consumption: core→socket)
    40→//   - AvgAggregation: Average values (e.g., temperature: core→socket)
    41→package metricstore
    42→
    43→import (
    44→	"fmt"
    45→	"time"
    46→
    47→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    48→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    49→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    50→)
    51→
    52→const (
    53→	DefaultMaxWorkers                 = 10
    54→	DefaultBufferCapacity             = 512
    55→	DefaultGCTriggerInterval          = 100
    56→	DefaultMemoryUsageTrackerInterval = 5 * time.Minute
    57→)
    58→
    59→// Checkpoints configures periodic persistence of in-memory metric data.
    60→//
    61→// Fields:
    62→//   - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal"
    63→//   - RootDir:    Filesystem path for checkpoint files (created if missing)
    64→type Checkpoints struct {
    65→	FileFormat string `json:"file-format"`
    66→	RootDir    string `json:"directory"`
    67→}
    68→
    69→// Debug provides development and profiling options.
    70→//
    71→// Fields:
    72→//   - DumpToFile: Path to dump checkpoint data for inspection (empty = disabled)
    73→//   - EnableGops: Enable gops agent for live runtime debugging (https://github.com/google/gops)
    74→type Debug struct {
    75→	DumpToFile string `json:"dump-to-file"`
    76→	EnableGops bool   `json:"gops"`
    77→}
    78→
    79→// Cleanup configures long-term storage of old metric data.
    80→//
    81→// Data older than RetentionInMemory is archived to disk or deleted.
    82→// The cleanup interval is always RetentionInMemory.
    83→//
    84→// Fields:
    85→//   - RootDir: Filesystem path for archived data (used in "archive" mode)
    86→//   - Mode:    "delete" (discard old data) or "archive" (write to RootDir)
    87→type Cleanup struct {
    88→	RootDir string `json:"directory"`
    89→	Mode    string `json:"mode"`
    90→}
    91→
    92→// Subscriptions defines NATS topics to subscribe to for metric ingestion.
    93→//
    94→// Each subscription receives metrics via NATS messaging, enabling real-time
    95→// data collection from compute nodes.
    96→//
    97→// Fields:
    98→//   - SubscribeTo: NATS subject/channel name (e.g., "metrics.compute.*")
    99→//   - ClusterTag:  Default cluster name for metrics without cluster tag (optional)
   100→type Subscriptions []struct {
   101→	// Channel name
   102→	SubscribeTo string `json:"subscribe-to"`
   103→
   104→	// Allow lines without a cluster tag, use this as default, optional
   105→	ClusterTag string `json:"cluster-tag"`
   106→}
   107→
   108→// MetricStoreConfig defines the main configuration for the metricstore.
   109→//
   110→// Loaded from cc-backend's config.json "metricstore" section. Controls memory usage,
   111→// persistence, archiving, and metric ingestion.
   112→//
   113→// Fields:
   114→//   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
   115→//   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
   116→//   - MemoryCap:         Max bytes for buffer data (0 = unlimited); triggers forceFree when exceeded
   117→//   - Checkpoints:       Periodic persistence configuration
   118→//   - Debug:             Development/profiling options (nil = disabled)
   119→//   - Archive:           Long-term storage configuration (nil = disabled)
   120→//   - Subscriptions:     NATS topics for metric ingestion (nil = polling only)
   121→type MetricStoreConfig struct {
   122→	// Number of concurrent workers for checkpoint and archive operations.
   123→	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
   124→	NumWorkers        int            `json:"num-workers"`
   125→	RetentionInMemory string         `json:"retention-in-memory"`
   126→	MemoryCap         int            `json:"memory-cap"`
   127→	Checkpoints       Checkpoints    `json:"checkpoints"`
   128→	Debug             *Debug         `json:"debug"`
   129→	Cleanup           *Cleanup       `json:"cleanup"`
   130→	Subscriptions     *Subscriptions `json:"nats-subscriptions"`
   131→}
   132→
   133→// Keys is the global metricstore configuration instance.
   134→//
   135→// Initialized with defaults, then overwritten by cc-backend's config.json.
   136→// Accessed by Init(), Checkpointing(), and other lifecycle functions.
   137→var Keys MetricStoreConfig = MetricStoreConfig{
   138→	Checkpoints: Checkpoints{
   139→		FileFormat: "wal",
   140→		RootDir:    "./var/checkpoints",
   141→	},
   142→	Cleanup: &Cleanup{
   143→		Mode: "delete",
   144→	},
   145→}
   146→
   147→// AggregationStrategy defines how to combine metric values across hierarchy levels.
   148→//
   149→// Used when transforming data from finer-grained scopes (e.g., core) to coarser scopes
   150→// (e.g., socket). This is SPATIAL aggregation, not TEMPORAL (time-based) aggregation.
   151→//
   152→// Values:
   153→//   - NoAggregation:  Do not aggregate (incompatible scopes or non-aggregatable metrics)
   154→//   - SumAggregation: Add values (e.g., power: sum core power → socket power)
   155→//   - AvgAggregation: Average values (e.g., temperature: average core temps → socket temp)
   156→type AggregationStrategy int
   157→
   158→const (
   159→	NoAggregation  AggregationStrategy = iota // Do not aggregate
   160→	SumAggregation                            // Sum values (e.g., power, energy)
   161→	AvgAggregation                            // Average values (e.g., temperature, utilization)
   162→)
   163→
   164→// AssignAggregationStrategy parses a string into an AggregationStrategy value.
   165→//
   166→// Used when loading metric configurations from JSON/YAML files.
   167→//
   168→// Parameters:
   169→//   - str: "sum", "avg", or "" (empty string for NoAggregation)
   170→//
   171→// Returns:
   172→//   - AggregationStrategy: Parsed value
   173→//   - error:               Non-nil if str is unrecognized
   174→func AssignAggregationStrategy(str string) (AggregationStrategy, error) {
   175→	switch str {
   176→	case "":
   177→		return NoAggregation, nil
   178→	case "sum":
   179→		return SumAggregation, nil
   180→	case "avg":
   181→		return AvgAggregation, nil
   182→	default:
   183→		return NoAggregation, fmt.Errorf("[METRICSTORE]> unknown aggregation strategy: %s", str)
   184→	}
   185→}
   186→
   187→// MetricConfig defines configuration for a single metric type.
   188→//
   189→// Stored in the global Metrics map, keyed by metric name (e.g., "cpu_load").
   190→//
   191→// Fields:
   192→//   - Frequency:   Measurement interval in seconds (e.g., 60 for 1-minute granularity)
   193→//   - Aggregation: How to combine values across hierarchy levels (sum/avg/none)
   194→//   - offset:      Internal index into Level.metrics slice (assigned during Init)
   195→type MetricConfig struct {
   196→	// Interval in seconds at which measurements are stored
   197→	Frequency int64
   198→
   199→	// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
   200→	Aggregation AggregationStrategy
   201→
   202→	// Private, used internally...
   203→	offset int
   204→}
   205→
   206→func BuildMetricList() map[string]MetricConfig {
   207→	var metrics map[string]MetricConfig = make(map[string]MetricConfig)
   208→
   209→	addMetric := func(name string, metric MetricConfig) error {
   210→		if metrics == nil {
   211→			metrics = make(map[string]MetricConfig, 0)
   212→		}
   213→
   214→		if existingMetric, ok := metrics[name]; ok {
   215→			if existingMetric.Frequency != metric.Frequency {
   216→				if existingMetric.Frequency < metric.Frequency {
   217→					existingMetric.Frequency = metric.Frequency
   218→					metrics[name] = existingMetric
   219→				}
   220→			}
   221→		} else {
   222→			metrics[name] = metric
   223→		}
   224→
   225→		return nil
   226→	}
   227→
   228→	// Helper function to add metric configuration
   229→	addMetricConfig := func(mc *schema.MetricConfig) {
   230→		agg, err := AssignAggregationStrategy(mc.Aggregation)
   231→		if err != nil {
   232→			cclog.Warnf("Could not find aggregation strategy for metric config '%s': %s", mc.Name, err.Error())
   233→		}
   234→
   235→		addMetric(mc.Name, MetricConfig{
   236→			Frequency:   int64(mc.Timestep),
   237→			Aggregation: agg,
   238→		})
   239→	}
   240→	for _, c := range archive.Clusters {
   241→		for _, mc := range c.MetricConfig {
   242→			addMetricConfig(mc)
   243→		}
   244→
   245→		for _, sc := range c.SubClusters {
   246→			for _, mc := range sc.MetricConfig {
   247→				addMetricConfig(mc)
   248→			}
   249→		}
   250→	}
   251→
   252→	return metrics
   253→}
   254→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides config.go: Configuration structures and metric management.
//
// # Configuration Hierarchy
//
// The metricstore package uses nested configuration structures:
//
//	MetricStoreConfig (Keys)
//	├─ NumWorkers: Parallel checkpoint/archive workers
//	├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval)
//	├─ MemoryCap: Memory limit in bytes (triggers forceFree)
//	├─ Checkpoints: Persistence configuration
//	│  ├─ FileFormat: "json" or "wal" (default: "wal")
//	│  └─ RootDir: Checkpoint storage path
//	├─ Cleanup: Long-term storage configuration (interval = RetentionInMemory)
//	│  ├─ RootDir: Archive storage path (archive mode only)
//	│  └─ Mode: "delete" or "archive"
//	├─ Debug: Development/debugging options
//	└─ Subscriptions: NATS topic subscriptions for metric ingestion
//
// # Metric Configuration
//
// Each metric (e.g., "cpu_load", "mem_used") has a MetricConfig entry in the global
// Metrics map, defining:
//
//   - Frequency: Measurement interval in seconds
//   - Aggregation: How to combine values (sum/avg/none) when transforming scopes
//   - offset: Internal index into Level.metrics slice (assigned during Init)
//
// # AggregationStrategy
//
// Determines how to combine metric values when aggregating from finer to coarser scopes:
//
//   - NoAggregation: Do not combine (incompatible scopes)
//   - SumAggregation: Add values (e.g., power consumption: core→socket)
//   - AvgAggregation: Average values (e.g., temperature: core→socket)
package metricstore

import (
	"fmt"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

const (
	DefaultMaxWorkers                 = 10
	DefaultBufferCapacity             = 512
	DefaultGCTriggerInterval          = 100
	DefaultMemoryUsageTrackerInterval = 5 * time.Minute
)

// Checkpoints configures periodic persistence of in-memory metric data.
//
// Fields:
//   - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal"
//   - RootDir:    Filesystem path for checkpoint files (created if missing)
type Checkpoints struct {
	FileFormat string `json:"file-format"`
	RootDir    string `json:"directory"`
}

// Debug provides development and profiling options.
//
// Fields:
//   - DumpToFile: Path to dump checkpoint data for inspection (empty = disabled)
//   - EnableGops: Enable gops agent for live runtime debugging (https://github.com/google/gops)
type Debug struct {
	DumpToFile string `json:"dump-to-file"`
	EnableGops bool   `json:"gops"`
}

// Cleanup configures long-term storage of old metric data.
//
// Data older than RetentionInMemory is archived to disk or deleted.
// The cleanup interval is always RetentionInMemory.
//
// Fields:
//   - RootDir: Filesystem path for archived data (used in "archive" mode)
//   - Mode:    "delete" (discard old data) or "archive" (write to RootDir)
type Cleanup struct {
	RootDir string `json:"directory"`
	Mode    string `json:"mode"`
}

// Subscriptions defines NATS topics to subscribe to for metric ingestion.
//
// Each subscription receives metrics via NATS messaging, enabling real-time
// data collection from compute nodes.
//
// Fields:
//   - SubscribeTo: NATS subject/channel name (e.g., "metrics.compute.*")
//   - ClusterTag:  Default cluster name for metrics without cluster tag (optional)
type Subscriptions []struct {
	// Channel name
	SubscribeTo string `json:"subscribe-to"`

	// Allow lines without a cluster tag, use this as default, optional
	ClusterTag string `json:"cluster-tag"`
}

// MetricStoreConfig defines the main configuration for the metricstore.
//
// Loaded from cc-backend's config.json "metricstore" section. Controls memory usage,
// persistence, archiving, and metric ingestion.
//
// Fields:
//   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
//   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
//   - MemoryCap:         Max bytes for buffer data (0 = unlimited); triggers forceFree when exceeded
//   - Checkpoints:       Periodic persistence configuration
//   - Debug:             Development/profiling options (nil = disabled)
//   - Archive:           Long-term storage configuration (nil = disabled)
//   - Subscriptions:     NATS topics for metric ingestion (nil = polling only)
type MetricStoreConfig struct {
	// Number of concurrent workers for checkpoint and archive operations.
	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
	NumWorkers        int            `json:"num-workers"`
	RetentionInMemory string         `json:"retention-in-memory"`
	MemoryCap         int            `json:"memory-cap"`
	Checkpoints       Checkpoints    `json:"checkpoints"`
	Debug             *Debug         `json:"debug"`
	Cleanup           *Cleanup       `json:"cleanup"`
	Subscriptions     *Subscriptions `json:"nats-subscriptions"`
}

// Keys is the global metricstore configuration instance.
//
// Initialized with defaults, then overwritten by cc-backend's config.json.
// Accessed by Init(), Checkpointing(), and other lifecycle functions.
var Keys MetricStoreConfig = MetricStoreConfig{
	Checkpoints: Checkpoints{
		FileFormat: "wal",
		RootDir:    "./var/checkpoints",
	},
	Cleanup: &Cleanup{
		Mode: "delete",
	},
}

// AggregationStrategy defines how to combine metric values across hierarchy levels.
//
// Used when transforming data from finer-grained scopes (e.g., core) to coarser scopes
// (e.g., socket). This is SPATIAL aggregation, not TEMPORAL (time-based) aggregation.
//
// Values:
//   - NoAggregation:  Do not aggregate (incompatible scopes or non-aggregatable metrics)
//   - SumAggregation: Add values (e.g., power: sum core power → socket power)
//   - AvgAggregation: Average values (e.g., temperature: average core temps → socket temp)
type AggregationStrategy int

const (
	NoAggregation  AggregationStrategy = iota // Do not aggregate
	SumAggregation                            // Sum values (e.g., power, energy)
	AvgAggregation                            // Average values (e.g., temperature, utilization)
)

// AssignAggregationStrategy parses a string into an AggregationStrategy value.
//
// Used when loading metric configurations from JSON/YAML files.
//
// Parameters:
//   - str: "sum", "avg", or "" (empty string for NoAggregation)
//
// Returns:
//   - AggregationStrategy: Parsed value
//   - error:               Non-nil if str is unrecognized
func AssignAggregationStrategy(str string) (AggregationStrategy, error) {
	switch str {
	case "":
		return NoAggregation, nil
	case "sum":
		return SumAggregation, nil
	case "avg":
		return AvgAggregation, nil
	default:
		return NoAggregation, fmt.Errorf("[METRICSTORE]> unknown aggregation strategy: %s", str)
	}
}

// MetricConfig defines configuration for a single metric type.
//
// Stored in the global Metrics map, keyed by metric name (e.g., "cpu_load").
//
// Fields:
//   - Frequency:   Measurement interval in seconds (e.g., 60 for 1-minute granularity)
//   - Aggregation: How to combine values across hierarchy levels (sum/avg/none)
//   - offset:      Internal index into Level.metrics slice (assigned during Init)
type MetricConfig struct {
	// Interval in seconds at which measurements are stored
	Frequency int64

	// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
	Aggregation AggregationStrategy

	// Private, used internally...
	offset int
}

func BuildMetricList() map[string]MetricConfig {
	var metrics map[string]MetricConfig = make(map[string]MetricConfig)

	addMetric := func(name string, metric MetricConfig) error {
		if metrics == nil {
			metrics = make(map[string]MetricConfig, 0)
		}

		if existingMetric, ok := metrics[name]; ok {
			if existingMetric.Frequency != metric.Frequency {
				if existingMetric.Frequency < metric.Frequency {
					existingMetric.Frequency = metric.Frequency
					metrics[name] = existingMetric
				}
			}
		} else {
			metrics[name] = metric
		}

		return nil
	}

	// Helper function to add metric configuration
	addMetricConfig := func(mc *schema.MetricConfig) {
		agg, err := AssignAggregationStrategy(mc.Aggregation)
		if err != nil {
			cclog.Warnf("Could not find aggregation strategy for metric config '%s': %s", mc.Name, err.Error())
		}

		addMetric(mc.Name, MetricConfig{
			Frequency:   int64(mc.Timestep),
			Aggregation: agg,
		})
	}
	for _, c := range archive.Clusters {
		for _, mc := range c.MetricConfig {
			addMetricConfig(mc)
		}

		for _, sc := range c.SubClusters {
			for _, mc := range sc.MetricConfig {
				addMetricConfig(mc)
			}
		}
	}

	return metrics
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements checkpoint persistence for the in-memory metric store.
     7→//
     8→// Checkpoints enable graceful restarts by periodically saving in-memory metric
     9→// data to disk in JSON or binary format. The checkpoint system:
    10→//
    11→// Key Features:
    12→//   - Periodic background checkpointing via the Checkpointing() worker
    13→//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
    14→//   - Parallel checkpoint creation and loading using worker pools
    15→//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
    16→//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
    17→//   - Only saves unarchived data (archived data is already persisted elsewhere)
    18→//   - GC optimization during loading to prevent excessive heap growth
    19→//
    20→// Checkpoint Workflow:
    21→//  1. Init() loads checkpoints within retention window at startup
    22→//  2. Checkpointing() worker periodically saves new data
    23→//  3. Shutdown() writes final checkpoint before exit
    24→//
    25→// File Organization:
    26→//
    27→//	checkpoints/
    28→//	  cluster1/
    29→//	    host001/
    30→//	      1234567890.json  (JSON format: full subtree snapshot)
    31→//	      1234567890.bin   (binary format: full subtree snapshot)
    32→//	      current.wal      (WAL format: append-only per-entry log)
    33→//	    host002/
    34→//	      ...
    35→package metricstore
    36→
    37→import (
    38→	"bufio"
    39→	"context"
    40→	"encoding/json"
    41→	"errors"
    42→	"fmt"
    43→	"io/fs"
    44→	"os"
    45→	"path"
    46→	"path/filepath"
    47→	"sort"
    48→	"strconv"
    49→	"strings"
    50→	"sync"
    51→	"sync/atomic"
    52→	"time"
    53→
    54→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    55→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    56→)
    57→
    58→const (
    59→	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
    60→	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
    61→	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
    62→)
    63→
    64→// CheckpointMetrics represents metric data in a checkpoint file.
    65→// Whenever the structure changes, update MarshalJSON as well!
    66→type CheckpointMetrics struct {
    67→	Data      []schema.Float `json:"data"`
    68→	Frequency int64          `json:"frequency"`
    69→	Start     int64          `json:"start"`
    70→}
    71→
    72→// CheckpointFile represents the hierarchical structure of a checkpoint file.
    73→// It mirrors the Level tree structure from the MemoryStore.
    74→type CheckpointFile struct {
    75→	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
    76→	Children map[string]*CheckpointFile    `json:"children"`
    77→	From     int64                         `json:"from"`
    78→	To       int64                         `json:"to"`
    79→}
    80→
    81→// lastCheckpoint tracks the timestamp of the last checkpoint creation.
    82→var (
    83→	lastCheckpoint   time.Time
    84→	lastCheckpointMu sync.Mutex
    85→)
    86→
    87→// Checkpointing starts a background worker that periodically saves metric data to disk.
    88→//
    89→// Checkpoints are written every 12 hours (hardcoded).
    90→//
    91→// Format behaviour:
    92→//   - "json": Periodic checkpointing every checkpointInterval
    93→//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
    94→func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
    95→	lastCheckpointMu.Lock()
    96→	lastCheckpoint = time.Now()
    97→	lastCheckpointMu.Unlock()
    98→
    99→	ms := GetMemoryStore()
   100→
   101→	wg.Go(func() {
   102→
   103→		const checkpointInterval = 12 * time.Hour
   104→		d := checkpointInterval
   105→
   106→		ticker := time.NewTicker(d)
   107→		defer ticker.Stop()
   108→
   109→		for {
   110→			select {
   111→			case <-ctx.Done():
   112→				return
   113→			case <-ticker.C:
   114→				lastCheckpointMu.Lock()
   115→				from := lastCheckpoint
   116→				lastCheckpointMu.Unlock()
   117→
   118→				now := time.Now()
   119→				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))
   120→
   121→				if Keys.Checkpoints.FileFormat == "wal" {
   122→					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   123→					if err != nil {
   124→						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
   125→					} else {
   126→						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
   127→						lastCheckpointMu.Lock()
   128→						lastCheckpoint = now
   129→						lastCheckpointMu.Unlock()
   130→						// Rotate WAL files for successfully checkpointed hosts.
   131→						RotateWALFiles(hostDirs)
   132→					}
   133→				} else {
   134→					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
   135→					if err != nil {
   136→						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
   137→					} else {
   138→						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
   139→						lastCheckpointMu.Lock()
   140→						lastCheckpoint = now
   141→						lastCheckpointMu.Unlock()
   142→					}
   143→				}
   144→			}
   145→		}
   146→	})
   147→}
   148→
   149→// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
   150→//
   151→// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
   152→// This method manually constructs JSON to avoid allocations and interface conversions.
   153→func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
   154→	buf := make([]byte, 0, 128+len(cm.Data)*8)
   155→	buf = append(buf, `{"frequency":`...)
   156→	buf = strconv.AppendInt(buf, cm.Frequency, 10)
   157→	buf = append(buf, `,"start":`...)
   158→	buf = strconv.AppendInt(buf, cm.Start, 10)
   159→	buf = append(buf, `,"data":[`...)
   160→	for i, x := range cm.Data {
   161→		if i != 0 {
   162→			buf = append(buf, ',')
   163→		}
   164→		if x.IsNaN() {
   165→			buf = append(buf, `null`...)
   166→		} else {
   167→			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
   168→		}
   169→	}
   170→	buf = append(buf, `]}`...)
   171→	return buf, nil
   172→}
   173→
   174→// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format).
   175→//
   176→// Metrics at root and cluster levels are skipped. One file per host is created.
   177→// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
   178→// at a time, allowing concurrent writes/reads to other hosts.
   179→//
   180→// Returns the number of checkpoint files created and any errors encountered.
   181→func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
   182→	// Pre-calculate capacity by counting cluster/host pairs
   183→	m.root.lock.RLock()
   184→	totalHosts := 0
   185→	for _, l1 := range m.root.children {
   186→		l1.lock.RLock()
   187→		totalHosts += len(l1.children)
   188→		l1.lock.RUnlock()
   189→	}
   190→	m.root.lock.RUnlock()
   191→
   192→	levels := make([]*Level, 0, totalHosts)
   193→	selectors := make([][]string, 0, totalHosts)
   194→
   195→	m.root.lock.RLock()
   196→	for sel1, l1 := range m.root.children {
   197→		l1.lock.RLock()
   198→		for sel2, l2 := range l1.children {
   199→			levels = append(levels, l2)
   200→			selectors = append(selectors, []string{sel1, sel2})
   201→		}
   202→		l1.lock.RUnlock()
   203→	}
   204→	m.root.lock.RUnlock()
   205→
   206→	type workItem struct {
   207→		level    *Level
   208→		dir      string
   209→		selector []string
   210→	}
   211→
   212→	n, errs := int32(0), int32(0)
   213→
   214→	var wg sync.WaitGroup
   215→	wg.Add(Keys.NumWorkers)
   216→	work := make(chan workItem, Keys.NumWorkers*2)
   217→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   218→		go func() {
   219→			defer wg.Done()
   220→
   221→			for workItem := range work {
   222→				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
   223→					if err == ErrNoNewArchiveData {
   224→						continue
   225→					}
   226→
   227→					cclog.Errorf("[METRICSTORE]> error while checkpointing %#v: %s", workItem.selector, err.Error())
   228→					atomic.AddInt32(&errs, 1)
   229→				} else {
   230→					atomic.AddInt32(&n, 1)
   231→				}
   232→			}
   233→		}()
   234→	}
   235→
   236→	for i := 0; i < len(levels); i++ {
   237→		dir := path.Join(dir, path.Join(selectors[i]...))
   238→		work <- workItem{
   239→			level:    levels[i],
   240→			dir:      dir,
   241→			selector: selectors[i],
   242→		}
   243→	}
   244→
   245→	close(work)
   246→	wg.Wait()
   247→
   248→	if errs > 0 {
   249→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   250→	}
   251→	return int(n), nil
   252→}
   253→
   254→// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
   255→// Skips metrics that are already archived. Returns nil if no unarchived data exists.
   256→func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
   257→	l.lock.RLock()
   258→	defer l.lock.RUnlock()
   259→
   260→	retval := &CheckpointFile{
   261→		From:     from,
   262→		To:       to,
   263→		Metrics:  make(map[string]*CheckpointMetrics),
   264→		Children: make(map[string]*CheckpointFile),
   265→	}
   266→
   267→	for metric, minfo := range m.Metrics {
   268→		b := l.metrics[minfo.offset]
   269→		if b == nil {
   270→			continue
   271→		}
   272→
   273→		allArchived := true
   274→		b.iterFromTo(from, to, func(b *buffer) error {
   275→			if !b.archived {
   276→				allArchived = false
   277→				return fmt.Errorf("stop") // Early termination signal
   278→			}
   279→			return nil
   280→		})
   281→
   282→		if allArchived {
   283→			continue
   284→		}
   285→
   286→		data := make([]schema.Float, (to-from)/b.frequency+1)
   287→		data, start, end, err := b.read(from, to, data)
   288→		if err != nil {
   289→			return nil, err
   290→		}
   291→
   292→		for i := int((end - start) / b.frequency); i < len(data); i++ {
   293→			data[i] = schema.NaN
   294→		}
   295→
   296→		retval.Metrics[metric] = &CheckpointMetrics{
   297→			Frequency: b.frequency,
   298→			Start:     start,
   299→			Data:      data,
   300→		}
   301→	}
   302→
   303→	for name, child := range l.children {
   304→		val, err := child.toCheckpointFile(from, to, m)
   305→		if err != nil {
   306→			return nil, err
   307→		}
   308→
   309→		if val != nil {
   310→			retval.Children[name] = val
   311→		}
   312→	}
   313→
   314→	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
   315→		return nil, nil
   316→	}
   317→
   318→	return retval, nil
   319→}
   320→
   321→// toCheckpoint writes a Level's data to a JSON checkpoint file.
   322→// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.
   323→func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
   324→	cf, err := l.toCheckpointFile(from, to, m)
   325→	if err != nil {
   326→		return err
   327→	}
   328→
   329→	if cf == nil {
   330→		return ErrNoNewArchiveData
   331→	}
   332→
   333→	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
   334→	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   335→	if err != nil && os.IsNotExist(err) {
   336→		err = os.MkdirAll(dir, CheckpointDirPerms)
   337→		if err == nil {
   338→			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
   339→		}
   340→	}
   341→	if err != nil {
   342→		return err
   343→	}
   344→	defer f.Close()
   345→
   346→	bw := bufio.NewWriter(f)
   347→	if err = json.NewEncoder(bw).Encode(cf); err != nil {
   348→		return err
   349→	}
   350→
   351→	return bw.Flush()
   352→}
   353→
   354→// enqueueCheckpointHosts traverses checkpoint directory and enqueues cluster/host pairs.
   355→// Returns error if directory structure is invalid.
   356→func enqueueCheckpointHosts(dir string, work chan<- [2]string) error {
   357→	clustersDir, err := os.ReadDir(dir)
   358→	if err != nil {
   359→		return err
   360→	}
   361→
   362→	for _, clusterDir := range clustersDir {
   363→		if !clusterDir.IsDir() {
   364→			return errors.New("[METRICSTORE]> expected only directories at first level of checkpoints/ directory")
   365→		}
   366→
   367→		hostsDir, err := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
   368→		if err != nil {
   369→			return err
   370→		}
   371→
   372→		for _, hostDir := range hostsDir {
   373→			if !hostDir.IsDir() {
   374→				return errors.New("[METRICSTORE]> expected only directories at second level of checkpoints/ directory")
   375→			}
   376→
   377→			work <- [2]string{clusterDir.Name(), hostDir.Name()}
   378→		}
   379→	}
   380→
   381→	return nil
   382→}
   383→
   384→// FromCheckpoint loads checkpoint files from disk into memory in parallel.
   385→//
   386→// Uses worker pool to load cluster/host combinations. Returns number of files
   387→// loaded and any errors.
   388→func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
   389→	var wg sync.WaitGroup
   390→	work := make(chan [2]string, Keys.NumWorkers*4)
   391→	n, errs := int32(0), int32(0)
   392→
   393→	wg.Add(Keys.NumWorkers)
   394→	for worker := 0; worker < Keys.NumWorkers; worker++ {
   395→		go func() {
   396→			defer wg.Done()
   397→			for host := range work {
   398→				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
   399→				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
   400→				if err != nil {
   401→					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
   402→					atomic.AddInt32(&errs, 1)
   403→				}
   404→				atomic.AddInt32(&n, int32(nn))
   405→			}
   406→		}()
   407→	}
   408→
   409→	err := enqueueCheckpointHosts(dir, work)
   410→	close(work)
   411→	wg.Wait()
   412→
   413→	if err != nil {
   414→		return int(n), err
   415→	}
   416→
   417→	if errs > 0 {
   418→		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
   419→	}
   420→	return int(n), nil
   421→}
   422→
   423→// FromCheckpointFiles is the main entry point for loading checkpoints at startup.
   424→//
   425→// Creates checkpoint directory if it doesn't exist. This function must be called
   426→// before any writes or reads, and can only be called once.
   427→func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
   428→	if _, err := os.Stat(dir); os.IsNotExist(err) {
   429→		err := os.MkdirAll(dir, CheckpointDirPerms)
   430→		if err != nil {
   431→			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
   432→		}
   433→		cclog.Debugf("[METRICSTORE]> %#v Directory created successfully", dir)
   434→	}
   435→
   436→	return m.FromCheckpoint(dir, from)
   437→}
   438→
   439→func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
   440→	br := bufio.NewReader(f)
   441→	cf := &CheckpointFile{}
   442→	if err := json.NewDecoder(br).Decode(cf); err != nil {
   443→		return err
   444→	}
   445→
   446→	if cf.To != 0 && cf.To < from {
   447→		return nil
   448→	}
   449→
   450→	if err := l.loadFile(cf, m); err != nil {
   451→		return err
   452→	}
   453→
   454→	return nil
   455→}
   456→
   457→func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
   458→	for name, metric := range cf.Metrics {
   459→		n := len(metric.Data)
   460→		b := &buffer{
   461→			frequency: metric.Frequency,
   462→			start:     metric.Start,
   463→			data:      metric.Data[0:n:n],
   464→			prev:      nil,
   465→			next:      nil,
   466→			archived:  true,
   467→		}
   468→
   469→		minfo, ok := m.Metrics[name]
   470→		if !ok {
   471→			continue
   472→		}
   473→
   474→		prev := l.metrics[minfo.offset]
   475→		if prev == nil {
   476→			l.metrics[minfo.offset] = b
   477→		} else {
   478→			if prev.start > b.start {
   479→				return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start)
   480→			}
   481→
   482→			b.prev = prev
   483→			prev.next = b
   484→		}
   485→		l.metrics[minfo.offset] = b
   486→	}
   487→
   488→	if len(cf.Children) > 0 && l.children == nil {
   489→		l.children = make(map[string]*Level)
   490→	}
   491→
   492→	for sel, childCf := range cf.Children {
   493→		child, ok := l.children[sel]
   494→		if !ok {
   495→			child = &Level{
   496→				metrics:  make([]*buffer, len(m.Metrics)),
   497→				children: nil,
   498→			}
   499→			l.children[sel] = child
   500→		}
   501→
   502→		if err := child.loadFile(childCf, m); err != nil {
   503→			return err
   504→		}
   505→	}
   506→
   507→	return nil
   508→}
   509→
   510→// fromCheckpoint loads all checkpoint files (JSON, binary snapshot, WAL) for a
   511→// single host directory. Snapshot files are loaded first (sorted by timestamp),
   512→// then current.wal is replayed on top.
   513→func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {
   514→	direntries, err := os.ReadDir(dir)
   515→	if err != nil {
   516→		if os.IsNotExist(err) {
   517→			return 0, nil
   518→		}
   519→		return 0, err
   520→	}
   521→
   522→	allFiles := make([]fs.DirEntry, 0)
   523→	var walEntry fs.DirEntry
   524→	filesLoaded := 0
   525→
   526→	for _, e := range direntries {
   527→		if e.IsDir() {
   528→			// Legacy: skip subdirectories (only used by old Avro format).
   529→			// These are ignored; their data is not loaded.
   530→			cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir)
   531→			continue
   532→		}
   533→
   534→		name := e.Name()
   535→		if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") {
   536→			allFiles = append(allFiles, e)
   537→		} else if name == "current.wal" {
   538→			walEntry = e
   539→		}
   540→		// Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes).
   541→	}
   542→
   543→	files, err := findFiles(allFiles, from, true)
   544→	if err != nil {
   545→		return filesLoaded, err
   546→	}
   547→
   548→	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
   549→		".json": l.loadJSONFile,
   550→		".bin":  l.loadBinaryFile,
   551→	}
   552→
   553→	for _, filename := range files {
   554→		ext := filepath.Ext(filename)
   555→		loader := loaders[ext]
   556→		if loader == nil {
   557→			cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename)
   558→			continue
   559→		}
   560→
   561→		err := func() error {
   562→			f, err := os.Open(path.Join(dir, filename))
   563→			if err != nil {
   564→				return err
   565→			}
   566→			defer f.Close()
   567→			return loader(m, f, from)
   568→		}()
   569→		if err != nil {
   570→			return filesLoaded, err
   571→		}
   572→		filesLoaded++
   573→	}
   574→
   575→	// Replay WAL after all snapshot files so it fills in data since the last snapshot.
   576→	if walEntry != nil {
   577→		err := func() error {
   578→			f, err := os.Open(path.Join(dir, walEntry.Name()))
   579→			if err != nil {
   580→				return err
   581→			}
   582→			defer f.Close()
   583→			return l.loadWALFile(m, f, from)
   584→		}()
   585→		if err != nil {
   586→			// WAL errors are non-fatal: the snapshot already loaded the bulk of data.
   587→			cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err)
   588→		} else {
   589→			filesLoaded++
   590→		}
   591→	}
   592→
   593→	return filesLoaded, nil
   594→}
   595→
   596→// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename.
   597→// Supports ".json" (format: "<ts>.json") and ".bin" (format: "<ts>.bin").
   598→func parseTimestampFromFilename(name string) (int64, error) {
   599→	switch {
   600→	case strings.HasSuffix(name, ".json"):
   601→		return strconv.ParseInt(name[:len(name)-5], 10, 64)
   602→	case strings.HasSuffix(name, ".bin"):
   603→		return strconv.ParseInt(name[:len(name)-4], 10, 64)
   604→	default:
   605→		return 0, fmt.Errorf("unknown checkpoint extension for file %q", name)
   606→	}
   607→}
   608→
   609→// findFiles returns filenames from direntries whose timestamps satisfy the filter.
   610→// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the
   611→// last file before t if t falls between two files).
   612→func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {
   613→	nums := map[string]int64{}
   614→	for _, e := range direntries {
   615→		name := e.Name()
   616→		if !strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".bin") {
   617→			continue
   618→		}
   619→
   620→		ts, err := parseTimestampFromFilename(name)
   621→		if err != nil {
   622→			return nil, err
   623→		}
   624→		nums[name] = ts
   625→	}
   626→
   627→	sort.Slice(direntries, func(i, j int) bool {
   628→		a, b := direntries[i], direntries[j]
   629→		return nums[a.Name()] < nums[b.Name()]
   630→	})
   631→
   632→	if len(nums) == 0 {
   633→		return nil, nil
   634→	}
   635→
   636→	filenames := make([]string, 0)
   637→
   638→	for i, e := range direntries {
   639→		ts1 := nums[e.Name()]
   640→
   641→		if findMoreRecentFiles && t <= ts1 {
   642→			filenames = append(filenames, e.Name())
   643→		} else if !findMoreRecentFiles && ts1 <= t && ts1 != 0 {
   644→			filenames = append(filenames, e.Name())
   645→		}
   646→
   647→		if i == len(direntries)-1 {
   648→			continue
   649→		}
   650→
   651→		enext := direntries[i+1]
   652→		ts2 := nums[enext.Name()]
   653→
   654→		if findMoreRecentFiles {
   655→			if ts1 < t && t < ts2 {
   656→				filenames = append(filenames, e.Name())
   657→			}
   658→		}
   659→	}
   660→
   661→	return filenames, nil
   662→}
   663→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements checkpoint persistence for the in-memory metric store.
//
// Checkpoints enable graceful restarts by periodically saving in-memory metric
// data to disk in JSON or binary format. The checkpoint system:
//
// Key Features:
//   - Periodic background checkpointing via the Checkpointing() worker
//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
//   - Parallel checkpoint creation and loading using worker pools
//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
//   - Only saves unarchived data (archived data is already persisted elsewhere)
//   - GC optimization during loading to prevent excessive heap growth
//
// Checkpoint Workflow:
//  1. Init() loads checkpoints within retention window at startup
//  2. Checkpointing() worker periodically saves new data
//  3. Shutdown() writes final checkpoint before exit
//
// File Organization:
//
//	checkpoints/
//	  cluster1/
//	    host001/
//	      1234567890.json  (JSON format: full subtree snapshot)
//	      1234567890.bin   (binary format: full subtree snapshot)
//	      current.wal      (WAL format: append-only per-entry log)
//	    host002/
//	      ...
package metricstore

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

const (
	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
)

// CheckpointMetrics represents metric data in a checkpoint file.
// Whenever the structure changes, update MarshalJSON as well!
type CheckpointMetrics struct {
	Data      []schema.Float `json:"data"`
	Frequency int64          `json:"frequency"`
	Start     int64          `json:"start"`
}

// CheckpointFile represents the hierarchical structure of a checkpoint file.
// It mirrors the Level tree structure from the MemoryStore.
type CheckpointFile struct {
	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
	Children map[string]*CheckpointFile    `json:"children"`
	From     int64                         `json:"from"`
	To       int64                         `json:"to"`
}

// lastCheckpoint tracks the timestamp of the last checkpoint creation.
var (
	lastCheckpoint   time.Time
	lastCheckpointMu sync.Mutex
)

// Checkpointing starts a background worker that periodically saves metric data to disk.
//
// Checkpoints are written every 12 hours (hardcoded).
//
// Format behaviour:
//   - "json": Periodic checkpointing every checkpointInterval
//   - "wal":  Periodic binary snapshots + WAL rotation every checkpointInterval
func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
	lastCheckpointMu.Lock()
	lastCheckpoint = time.Now()
	lastCheckpointMu.Unlock()

	ms := GetMemoryStore()

	wg.Go(func() {

		const checkpointInterval = 12 * time.Hour
		d := checkpointInterval

		ticker := time.NewTicker(d)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				lastCheckpointMu.Lock()
				from := lastCheckpoint
				lastCheckpointMu.Unlock()

				now := time.Now()
				cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", from.Format(time.RFC3339))

				if Keys.Checkpoints.FileFormat == "wal" {
					n, hostDirs, err := ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> binary checkpointing failed: %s", err.Error())
					} else {
						cclog.Infof("[METRICSTORE]> done: %d binary snapshot files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
						// Rotate WAL files for successfully checkpointed hosts.
						RotateWALFiles(hostDirs)
					}
				} else {
					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), now.Unix())
					if err != nil {
						cclog.Errorf("[METRICSTORE]> checkpointing failed: %s", err.Error())
					} else {
						cclog.Infof("[METRICSTORE]> done: %d checkpoint files created", n)
						lastCheckpointMu.Lock()
						lastCheckpoint = now
						lastCheckpointMu.Unlock()
					}
				}
			}
		}
	})
}

// MarshalJSON provides optimized JSON encoding for CheckpointMetrics.
//
// Since schema.Float has custom MarshalJSON, serializing []Float has significant overhead.
// This method manually constructs JSON to avoid allocations and interface conversions.
func (cm *CheckpointMetrics) MarshalJSON() ([]byte, error) {
	buf := make([]byte, 0, 128+len(cm.Data)*8)
	buf = append(buf, `{"frequency":`...)
	buf = strconv.AppendInt(buf, cm.Frequency, 10)
	buf = append(buf, `,"start":`...)
	buf = strconv.AppendInt(buf, cm.Start, 10)
	buf = append(buf, `,"data":[`...)
	for i, x := range cm.Data {
		if i != 0 {
			buf = append(buf, ',')
		}
		if x.IsNaN() {
			buf = append(buf, `null`...)
		} else {
			buf = strconv.AppendFloat(buf, float64(x), 'f', 1, 32)
		}
	}
	buf = append(buf, `]}`...)
	return buf, nil
}

// ToCheckpoint writes metric data to checkpoint files in parallel (JSON format).
//
// Metrics at root and cluster levels are skipped. One file per host is created.
// Uses worker pool (Keys.NumWorkers) for parallel processing. Only locks one host
// at a time, allowing concurrent writes/reads to other hosts.
//
// Returns the number of checkpoint files created and any errors encountered.
func (m *MemoryStore) ToCheckpoint(dir string, from, to int64) (int, error) {
	// Pre-calculate capacity by counting cluster/host pairs
	m.root.lock.RLock()
	totalHosts := 0
	for _, l1 := range m.root.children {
		l1.lock.RLock()
		totalHosts += len(l1.children)
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	levels := make([]*Level, 0, totalHosts)
	selectors := make([][]string, 0, totalHosts)

	m.root.lock.RLock()
	for sel1, l1 := range m.root.children {
		l1.lock.RLock()
		for sel2, l2 := range l1.children {
			levels = append(levels, l2)
			selectors = append(selectors, []string{sel1, sel2})
		}
		l1.lock.RUnlock()
	}
	m.root.lock.RUnlock()

	type workItem struct {
		level    *Level
		dir      string
		selector []string
	}

	n, errs := int32(0), int32(0)

	var wg sync.WaitGroup
	wg.Add(Keys.NumWorkers)
	work := make(chan workItem, Keys.NumWorkers*2)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()

			for workItem := range work {
				if err := workItem.level.toCheckpoint(workItem.dir, from, to, m); err != nil {
					if err == ErrNoNewArchiveData {
						continue
					}

					cclog.Errorf("[METRICSTORE]> error while checkpointing %#v: %s", workItem.selector, err.Error())
					atomic.AddInt32(&errs, 1)
				} else {
					atomic.AddInt32(&n, 1)
				}
			}
		}()
	}

	for i := 0; i < len(levels); i++ {
		dir := path.Join(dir, path.Join(selectors[i]...))
		work <- workItem{
			level:    levels[i],
			dir:      dir,
			selector: selectors[i],
		}
	}

	close(work)
	wg.Wait()

	if errs > 0 {
		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
	}
	return int(n), nil
}

// toCheckpointFile recursively converts a Level tree to CheckpointFile structure.
// Skips metrics that are already archived. Returns nil if no unarchived data exists.
func (l *Level) toCheckpointFile(from, to int64, m *MemoryStore) (*CheckpointFile, error) {
	l.lock.RLock()
	defer l.lock.RUnlock()

	retval := &CheckpointFile{
		From:     from,
		To:       to,
		Metrics:  make(map[string]*CheckpointMetrics),
		Children: make(map[string]*CheckpointFile),
	}

	for metric, minfo := range m.Metrics {
		b := l.metrics[minfo.offset]
		if b == nil {
			continue
		}

		allArchived := true
		b.iterFromTo(from, to, func(b *buffer) error {
			if !b.archived {
				allArchived = false
				return fmt.Errorf("stop") // Early termination signal
			}
			return nil
		})

		if allArchived {
			continue
		}

		data := make([]schema.Float, (to-from)/b.frequency+1)
		data, start, end, err := b.read(from, to, data)
		if err != nil {
			return nil, err
		}

		for i := int((end - start) / b.frequency); i < len(data); i++ {
			data[i] = schema.NaN
		}

		retval.Metrics[metric] = &CheckpointMetrics{
			Frequency: b.frequency,
			Start:     start,
			Data:      data,
		}
	}

	for name, child := range l.children {
		val, err := child.toCheckpointFile(from, to, m)
		if err != nil {
			return nil, err
		}

		if val != nil {
			retval.Children[name] = val
		}
	}

	if len(retval.Children) == 0 && len(retval.Metrics) == 0 {
		return nil, nil
	}

	return retval, nil
}

// toCheckpoint writes a Level's data to a JSON checkpoint file.
// Creates directory if needed. Returns ErrNoNewArchiveData if nothing to save.
func (l *Level) toCheckpoint(dir string, from, to int64, m *MemoryStore) error {
	cf, err := l.toCheckpointFile(from, to, m)
	if err != nil {
		return err
	}

	if cf == nil {
		return ErrNoNewArchiveData
	}

	filepath := path.Join(dir, fmt.Sprintf("%d.json", from))
	f, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
	if err != nil && os.IsNotExist(err) {
		err = os.MkdirAll(dir, CheckpointDirPerms)
		if err == nil {
			f, err = os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, CheckpointFilePerms)
		}
	}
	if err != nil {
		return err
	}
	defer f.Close()

	bw := bufio.NewWriter(f)
	if err = json.NewEncoder(bw).Encode(cf); err != nil {
		return err
	}

	return bw.Flush()
}

// enqueueCheckpointHosts traverses checkpoint directory and enqueues cluster/host pairs.
// Returns error if directory structure is invalid.
func enqueueCheckpointHosts(dir string, work chan<- [2]string) error {
	clustersDir, err := os.ReadDir(dir)
	if err != nil {
		return err
	}

	for _, clusterDir := range clustersDir {
		if !clusterDir.IsDir() {
			return errors.New("[METRICSTORE]> expected only directories at first level of checkpoints/ directory")
		}

		hostsDir, err := os.ReadDir(filepath.Join(dir, clusterDir.Name()))
		if err != nil {
			return err
		}

		for _, hostDir := range hostsDir {
			if !hostDir.IsDir() {
				return errors.New("[METRICSTORE]> expected only directories at second level of checkpoints/ directory")
			}

			work <- [2]string{clusterDir.Name(), hostDir.Name()}
		}
	}

	return nil
}

// FromCheckpoint loads checkpoint files from disk into memory in parallel.
//
// Uses worker pool to load cluster/host combinations. Returns number of files
// loaded and any errors.
func (m *MemoryStore) FromCheckpoint(dir string, from int64) (int, error) {
	var wg sync.WaitGroup
	work := make(chan [2]string, Keys.NumWorkers*4)
	n, errs := int32(0), int32(0)

	wg.Add(Keys.NumWorkers)
	for worker := 0; worker < Keys.NumWorkers; worker++ {
		go func() {
			defer wg.Done()
			for host := range work {
				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
					atomic.AddInt32(&errs, 1)
				}
				atomic.AddInt32(&n, int32(nn))
			}
		}()
	}

	err := enqueueCheckpointHosts(dir, work)
	close(work)
	wg.Wait()

	if err != nil {
		return int(n), err
	}

	if errs > 0 {
		return int(n), fmt.Errorf("[METRICSTORE]> %d errors happened while creating checkpoints (%d successes)", errs, n)
	}
	return int(n), nil
}

// FromCheckpointFiles is the main entry point for loading checkpoints at startup.
//
// Creates checkpoint directory if it doesn't exist. This function must be called
// before any writes or reads, and can only be called once.
func (m *MemoryStore) FromCheckpointFiles(dir string, from int64) (int, error) {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, CheckpointDirPerms)
		if err != nil {
			cclog.Fatalf("[METRICSTORE]> Error creating directory: %#v\n", err)
		}
		cclog.Debugf("[METRICSTORE]> %#v Directory created successfully", dir)
	}

	return m.FromCheckpoint(dir, from)
}

func (l *Level) loadJSONFile(m *MemoryStore, f *os.File, from int64) error {
	br := bufio.NewReader(f)
	cf := &CheckpointFile{}
	if err := json.NewDecoder(br).Decode(cf); err != nil {
		return err
	}

	if cf.To != 0 && cf.To < from {
		return nil
	}

	if err := l.loadFile(cf, m); err != nil {
		return err
	}

	return nil
}

func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
	for name, metric := range cf.Metrics {
		n := len(metric.Data)
		b := &buffer{
			frequency: metric.Frequency,
			start:     metric.Start,
			data:      metric.Data[0:n:n],
			prev:      nil,
			next:      nil,
			archived:  true,
		}

		minfo, ok := m.Metrics[name]
		if !ok {
			continue
		}

		prev := l.metrics[minfo.offset]
		if prev == nil {
			l.metrics[minfo.offset] = b
		} else {
			if prev.start > b.start {
				return fmt.Errorf("[METRICSTORE]> buffer start time %d is before previous buffer start %d", b.start, prev.start)
			}

			b.prev = prev
			prev.next = b
		}
		l.metrics[minfo.offset] = b
	}

	if len(cf.Children) > 0 && l.children == nil {
		l.children = make(map[string]*Level)
	}

	for sel, childCf := range cf.Children {
		child, ok := l.children[sel]
		if !ok {
			child = &Level{
				metrics:  make([]*buffer, len(m.Metrics)),
				children: nil,
			}
			l.children[sel] = child
		}

		if err := child.loadFile(childCf, m); err != nil {
			return err
		}
	}

	return nil
}

// fromCheckpoint loads all checkpoint files (JSON, binary snapshot, WAL) for a
// single host directory. Snapshot files are loaded first (sorted by timestamp),
// then current.wal is replayed on top.
func (l *Level) fromCheckpoint(m *MemoryStore, dir string, from int64) (int, error) {
	direntries, err := os.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return 0, nil
		}
		return 0, err
	}

	allFiles := make([]fs.DirEntry, 0)
	var walEntry fs.DirEntry
	filesLoaded := 0

	for _, e := range direntries {
		if e.IsDir() {
			// Legacy: skip subdirectories (only used by old Avro format).
			// These are ignored; their data is not loaded.
			cclog.Debugf("[METRICSTORE]> skipping subdirectory %s in checkpoint dir %s", e.Name(), dir)
			continue
		}

		name := e.Name()
		if strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".bin") {
			allFiles = append(allFiles, e)
		} else if name == "current.wal" {
			walEntry = e
		}
		// Silently ignore other files (e.g., .tmp, .bin.tmp from interrupted writes).
	}

	files, err := findFiles(allFiles, from, true)
	if err != nil {
		return filesLoaded, err
	}

	loaders := map[string]func(*MemoryStore, *os.File, int64) error{
		".json": l.loadJSONFile,
		".bin":  l.loadBinaryFile,
	}

	for _, filename := range files {
		ext := filepath.Ext(filename)
		loader := loaders[ext]
		if loader == nil {
			cclog.Warnf("[METRICSTORE]> unknown extension for checkpoint file %s", filename)
			continue
		}

		err := func() error {
			f, err := os.Open(path.Join(dir, filename))
			if err != nil {
				return err
			}
			defer f.Close()
			return loader(m, f, from)
		}()
		if err != nil {
			return filesLoaded, err
		}
		filesLoaded++
	}

	// Replay WAL after all snapshot files so it fills in data since the last snapshot.
	if walEntry != nil {
		err := func() error {
			f, err := os.Open(path.Join(dir, walEntry.Name()))
			if err != nil {
				return err
			}
			defer f.Close()
			return l.loadWALFile(m, f, from)
		}()
		if err != nil {
			// WAL errors are non-fatal: the snapshot already loaded the bulk of data.
			cclog.Warnf("[METRICSTORE]> WAL replay error for %s: %v (data since last snapshot may be missing)", dir, err)
		} else {
			filesLoaded++
		}
	}

	return filesLoaded, nil
}

// parseTimestampFromFilename extracts a Unix timestamp from a checkpoint filename.
// Supports ".json" (format: "<ts>.json") and ".bin" (format: "<ts>.bin").
func parseTimestampFromFilename(name string) (int64, error) {
	switch {
	case strings.HasSuffix(name, ".json"):
		return strconv.ParseInt(name[:len(name)-5], 10, 64)
	case strings.HasSuffix(name, ".bin"):
		return strconv.ParseInt(name[:len(name)-4], 10, 64)
	default:
		return 0, fmt.Errorf("unknown checkpoint extension for file %q", name)
	}
}

// findFiles returns filenames from direntries whose timestamps satisfy the filter.
// If findMoreRecentFiles is true, returns files with timestamps >= t (plus the
// last file before t if t falls between two files).
func findFiles(direntries []fs.DirEntry, t int64, findMoreRecentFiles bool) ([]string, error) {
	nums := map[string]int64{}
	for _, e := range direntries {
		name := e.Name()
		if !strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".bin") {
			continue
		}

		ts, err := parseTimestampFromFilename(name)
		if err != nil {
			return nil, err
		}
		nums[name] = ts
	}

	sort.Slice(direntries, func(i, j int) bool {
		a, b := direntries[i], direntries[j]
		return nums[a.Name()] < nums[b.Name()]
	})

	if len(nums) == 0 {
		return nil, nil
	}

	filenames := make([]string, 0)

	for i, e := range direntries {
		ts1 := nums[e.Name()]

		if findMoreRecentFiles && t <= ts1 {
			filenames = append(filenames, e.Name())
		} else if !findMoreRecentFiles && ts1 <= t && ts1 != 0 {
			filenames = append(filenames, e.Name())
		}

		if i == len(direntries)-1 {
			continue
		}

		enext := direntries[i+1]
		ts2 := nums[enext.Name()]

		if findMoreRecentFiles {
			if ts1 < t && t < ts2 {
				filenames = append(filenames, e.Name())
			}
		}
	}

	return filenames, nil
}


Now I have all the context. Let me apply all the fixes.


Edit

	for key, child := range l.children {
		m, err := child.free(t)
		n += m
		if err != nil {
			return n, err
		}
		if child.isEmpty() {
			delete(l.children, key)
		}
	}

	return n, nil
}

	for key, child := range l.children {
		m, empty, err := child.freeAndCheckEmpty(t)
		n += m
		if err != nil {
			return n, err
		}
		if empty {
			delete(l.children, key)
		}
	}

	return n, nil
}