     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricdispatch provides a unified interface for loading and caching job metric data.
     7→//
     8→// This package serves as a central dispatcher that routes metric data requests to the appropriate
     9→// backend based on job state. For running jobs, data is fetched from the metric store (e.g., cc-metric-store).
    10→// For completed jobs, data is retrieved from the file-based job archive.
    11→//
    12→// # Key Features
    13→//
    14→//   - Automatic backend selection based on job state (running vs. archived)
    15→//   - LRU cache for performance optimization (128 MB default cache size)
    16→//   - Data resampling using Largest Triangle Three Bucket algorithm for archived data
    17→//   - Automatic statistics series generation for jobs with many nodes
    18→//   - Support for scoped metrics (node, socket, accelerator, core)
    19→//
    20→// # Cache Behavior
    21→//
    22→// Cached data has different TTL (time-to-live) values depending on job state:
    23→//   - Running jobs: 2 minutes (data changes frequently)
    24→//   - Completed jobs: 5 hours (data is static)
    25→//
    26→// The cache key is based on job ID, state, requested metrics, scopes, and resolution.
    27→//
    28→// # Usage
    29→//
    30→// The primary entry point is LoadData, which automatically handles both running and archived jobs:
    31→//
    32→//	jobData, err := metricdispatch.LoadData(job, metrics, scopes, ctx, resolution)
    33→//	if err != nil {
    34→//	    // Handle error
    35→//	}
    36→//
    37→// For statistics only, use LoadJobStats, LoadScopedJobStats, or LoadAverages depending on the required format.
    38→package metricdispatch
    39→
    40→import (
    41→	"context"
    42→	"fmt"
    43→	"math"
    44→	"time"
    45→
    46→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    47→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    48→	"github.com/ClusterCockpit/cc-lib/v2/lrucache"
    49→	"github.com/ClusterCockpit/cc-lib/v2/resampler"
    50→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    51→)
    52→
    53→// cache is an LRU cache with 128 MB capacity for storing loaded job metric data.
    54→// The cache reduces load on both the metric store and archive backends.
    55→var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024)
    56→
    57→// cacheKey generates a unique cache key for a job's metric data based on job ID, state,
    58→// requested metrics, scopes, and resolution. Duration and StartTime are intentionally excluded
    59→// because job.ID is more unique and the cache TTL ensures entries don't persist indefinitely.
    60→func cacheKey(
    61→	job *schema.Job,
    62→	metrics []string,
    63→	scopes []schema.MetricScope,
    64→	resolution int,
    65→) string {
    66→	return fmt.Sprintf("%d(%s):[%v],[%v]-%d",
    67→		*job.ID, job.State, metrics, scopes, resolution)
    68→}
    69→
    70→// LoadData retrieves metric data for a job from the appropriate backend (memory store for running jobs,
    71→// archive for completed jobs) and applies caching, resampling, and statistics generation as needed.
    72→//
    73→// For running jobs or when archive is disabled, data is fetched from the metric store.
    74→// For completed archived jobs, data is loaded from the job archive and resampled if needed.
    75→//
    76→// Parameters:
    77→//   - job: The job for which to load metric data
    78→//   - metrics: List of metric names to load (nil loads all metrics for the cluster)
    79→//   - scopes: Metric scopes to include (nil defaults to node scope)
    80→//   - ctx: Context for cancellation and timeouts
    81→//   - resolution: Target number of data points for resampling (only applies to archived data)
    82→//
    83→// Returns the loaded job data and any error encountered. For partial errors (some metrics failed),
    84→// the function returns the successfully loaded data with a warning logged.
    85→func LoadData(job *schema.Job,
    86→	metrics []string,
    87→	scopes []schema.MetricScope,
    88→	ctx context.Context,
    89→	resolution int,
    90→) (schema.JobData, error) {
    91→	data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ any, ttl time.Duration, size int) {
    92→		var jd schema.JobData
    93→		var err error
    94→
    95→		if job.State == schema.JobStateRunning ||
    96→			job.MonitoringStatus == schema.MonitoringStatusRunningOrArchiving {
    97→
    98→			ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
    99→			if err != nil {
   100→				cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
   101→					job.Cluster, job.SubCluster, err.Error())
   102→				return err, 0, 0
   103→			}
   104→
   105→			if scopes == nil {
   106→				scopes = append(scopes, schema.MetricScopeNode)
   107→			}
   108→
   109→			if metrics == nil {
   110→				cluster := archive.GetCluster(job.Cluster)
   111→				for _, mc := range cluster.MetricConfig {
   112→					metrics = append(metrics, mc.Name)
   113→				}
   114→			}
   115→
   116→			jd, err = ms.LoadData(job, metrics, scopes, ctx, resolution)
   117→			if err != nil {
   118→				if len(jd) != 0 {
   119→					cclog.Warnf("partial error loading metrics from store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
   120→						job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
   121→				} else {
   122→					cclog.Warnf("failed to load job data from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
   123→						job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
   124→					return err, 0, 0
   125→				}
   126→			}
   127→			size = jd.Size()
   128→		} else {
   129→			var jdTemp schema.JobData
   130→			jdTemp, err = archive.GetHandle().LoadJobData(job)
   131→			if err != nil {
   132→				cclog.Warnf("failed to load job data from archive for job %d (user: %s, project: %s, cluster: %s-%s): %s",
   133→					job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
   134→				return err, 0, 0
   135→			}
   136→
   137→			jd = deepCopy(jdTemp)
   138→
   139→			// Resample archived data using Largest Triangle Three Bucket algorithm to reduce data points
   140→			// to the requested resolution, improving transfer performance and client-side rendering.
   141→			for _, v := range jd {
   142→				for _, v_ := range v {
   143→					timestep := int64(0)
   144→					for i := 0; i < len(v_.Series); i += 1 {
   145→						v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))
   146→						if err != nil {
   147→							return err, 0, 0
   148→						}
   149→					}
   150→					v_.Timestep = int(timestep)
   151→				}
   152→			}
   153→
   154→			// Filter job data to only include requested metrics and scopes, avoiding unnecessary data transfer.
   155→			if metrics != nil || scopes != nil {
   156→				if metrics == nil {
   157→					metrics = make([]string, 0, len(jd))
   158→					for k := range jd {
   159→						metrics = append(metrics, k)
   160→					}
   161→				}
   162→
   163→				res := schema.JobData{}
   164→				for _, metric := range metrics {
   165→					if perscope, ok := jd[metric]; ok {
   166→						if len(perscope) > 1 {
   167→							subset := make(map[schema.MetricScope]*schema.JobMetric)
   168→							for _, scope := range scopes {
   169→								if jm, ok := perscope[scope]; ok {
   170→									subset[scope] = jm
   171→								}
   172→							}
   173→
   174→							if len(subset) > 0 {
   175→								perscope = subset
   176→							}
   177→						}
   178→
   179→						res[metric] = perscope
   180→					}
   181→				}
   182→				jd = res
   183→			}
   184→			size = jd.Size()
   185→		}
   186→
   187→		ttl = 5 * time.Hour
   188→		if job.State == schema.JobStateRunning {
   189→			ttl = 2 * time.Minute
   190→		}
   191→
   192→		// Generate statistics series for jobs with many nodes to enable min/median/max graphs
   193→		// instead of overwhelming the UI with individual node lines. Note that newly calculated
   194→		// statistics use min/median/max, while archived statistics may use min/mean/max.
   195→		const maxSeriesSize int = 8
   196→		for _, scopes := range jd {
   197→			for _, jm := range scopes {
   198→				if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
   199→					continue
   200→				}
   201→
   202→				jm.AddStatisticsSeries()
   203→			}
   204→		}
   205→
   206→		nodeScopeRequested := false
   207→		for _, scope := range scopes {
   208→			if scope == schema.MetricScopeNode {
   209→				nodeScopeRequested = true
   210→			}
   211→		}
   212→
   213→		if nodeScopeRequested {
   214→			jd.AddNodeScope("flops_any")
   215→			jd.AddNodeScope("mem_bw")
   216→		}
   217→
   218→		// Round Resulting Stat Values
   219→		jd.RoundMetricStats()
   220→
   221→		return jd, ttl, size
   222→	})
   223→
   224→	if err, ok := data.(error); ok {
   225→		cclog.Errorf("error in cached dataset for job %d: %s", job.JobID, err.Error())
   226→		return nil, err
   227→	}
   228→
   229→	return data.(schema.JobData), nil
   230→}
   231→
   232→// LoadAverages computes average values for the specified metrics across all nodes of a job.
   233→// For running jobs, it loads statistics from the metric store. For completed jobs, it uses
   234→// the pre-calculated averages from the job archive. The results are appended to the data slice.
   235→func LoadAverages(
   236→	job *schema.Job,
   237→	metrics []string,
   238→	data [][]schema.Float,
   239→	ctx context.Context,
   240→) error {
   241→	if job.State != schema.JobStateRunning {
   242→		return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here?
   243→	}
   244→
   245→	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
   246→	if err != nil {
   247→		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
   248→			job.Cluster, job.SubCluster, err.Error())
   249→		return err
   250→	}
   251→
   252→	stats, err := ms.LoadStats(job, metrics, ctx)
   253→	if err != nil {
   254→		cclog.Warnf("failed to load statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
   255→			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
   256→		return err
   257→	}
   258→
   259→	for i, m := range metrics {
   260→		nodes, ok := stats[m]
   261→		if !ok {
   262→			data[i] = append(data[i], schema.NaN)
   263→			continue
   264→		}
   265→
   266→		sum := 0.0
   267→		for _, node := range nodes {
   268→			sum += node.Avg
   269→		}
   270→		data[i] = append(data[i], schema.Float(sum))
   271→	}
   272→
   273→	return nil
   274→}
   275→
   276→// LoadScopedJobStats retrieves job statistics organized by metric scope (node, socket, core, accelerator).
   277→// For running jobs, statistics are computed from the metric store. For completed jobs, pre-calculated
   278→// statistics are loaded from the job archive.
   279→func LoadScopedJobStats(
   280→	job *schema.Job,
   281→	metrics []string,
   282→	scopes []schema.MetricScope,
   283→	ctx context.Context,
   284→) (schema.ScopedJobStats, error) {
   285→	if job.State != schema.JobStateRunning {
   286→		return archive.LoadScopedStatsFromArchive(job, metrics, scopes)
   287→	}
   288→
   289→	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
   290→	if err != nil {
   291→		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
   292→			job.Cluster, job.SubCluster, err.Error())
   293→		return nil, err
   294→	}
   295→
   296→	scopedStats, err := ms.LoadScopedStats(job, metrics, scopes, ctx)
   297→	if err != nil {
   298→		cclog.Warnf("failed to load scoped statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
   299→			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
   300→		return nil, err
   301→	}
   302→
   303→	// Round Resulting Stat Values
   304→	scopedStats.RoundScopedMetricStats()
   305→
   306→	return scopedStats, nil
   307→}
   308→
   309→// LoadJobStats retrieves aggregated statistics (min/avg/max) for each requested metric across all job nodes.
   310→// For running jobs, statistics are computed from the metric store. For completed jobs, pre-calculated
   311→// statistics are loaded from the job archive.
   312→func LoadJobStats(
   313→	job *schema.Job,
   314→	metrics []string,
   315→	ctx context.Context,
   316→) (map[string]schema.MetricStatistics, error) {
   317→	if job.State != schema.JobStateRunning {
   318→		return archive.LoadStatsFromArchive(job, metrics)
   319→	}
   320→
   321→	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
   322→	if err != nil {
   323→		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
   324→			job.Cluster, job.SubCluster, err.Error())
   325→		return nil, err
   326→	}
   327→
   328→	data := make(map[string]schema.MetricStatistics, len(metrics))
   329→
   330→	stats, err := ms.LoadStats(job, metrics, ctx)
   331→	if err != nil {
   332→		cclog.Warnf("failed to load statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
   333→			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
   334→		return data, err
   335→	}
   336→
   337→	for _, m := range metrics {
   338→		sum, avg, min, max := 0.0, 0.0, 0.0, 0.0
   339→		nodes, ok := stats[m]
   340→		if !ok {
   341→			data[m] = schema.MetricStatistics{Min: min, Avg: avg, Max: max}
   342→			continue
   343→		}
   344→
   345→		for _, node := range nodes {
   346→			sum += node.Avg
   347→			min = math.Min(min, node.Min)
   348→			max = math.Max(max, node.Max)
   349→		}
   350→
   351→		data[m] = schema.MetricStatistics{
   352→			Avg: (math.Round((sum/float64(job.NumNodes))*100) / 100),
   353→			Min: (math.Round(min*100) / 100),
   354→			Max: (math.Round(max*100) / 100),
   355→		}
   356→	}
   357→
   358→	return data, nil
   359→}
   360→
   361→// LoadNodeData retrieves metric data for specific nodes in a cluster within a time range.
   362→// This is used for node monitoring views and system status pages. Data is always fetched from
   363→// the metric store (not the archive) since it's for current/recent node status monitoring.
   364→//
   365→// Returns a nested map structure: node -> metric -> scoped data.
   366→// FIXME: Add support for subcluster specific cc-metric-stores
   367→func LoadNodeData(
   368→	cluster string,
   369→	metrics, nodes []string,
   370→	scopes []schema.MetricScope,
   371→	from, to time.Time,
   372→	ctx context.Context,
   373→) (map[string]map[string][]*schema.JobMetric, error) {
   374→	if metrics == nil {
   375→		for _, m := range archive.GetCluster(cluster).MetricConfig {
   376→			metrics = append(metrics, m.Name)
   377→		}
   378→	}
   379→
   380→	ms, err := GetMetricDataRepo(cluster, "")
   381→	if err != nil {
   382→		cclog.Errorf("failed to access metricDataRepo for cluster %s: %s",
   383→			cluster, err.Error())
   384→		return nil, err
   385→	}
   386→
   387→	data, err := ms.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
   388→	if err != nil {
   389→		if len(data) != 0 {
   390→			cclog.Warnf("partial error loading node data from metric store for cluster %s: %s", cluster, err.Error())
   391→		} else {
   392→			cclog.Warnf("failed to load node data from metric store for cluster %s: %s", cluster, err.Error())
   393→			return nil, err
   394→		}
   395→	}
   396→
   397→	if data == nil {
   398→		return nil, fmt.Errorf("metric store for cluster '%s' does not support node data queries", cluster)
   399→	}
   400→
   401→	return data, nil
   402→}
   403→
   404→// LoadNodeListData retrieves time-series metric data for multiple nodes within a time range,
   405→// with optional resampling and automatic statistics generation for large datasets.
   406→// This is used for comparing multiple nodes or displaying node status over time.
   407→//
   408→// Returns a map of node names to their job-like metric data structures.
   409→func LoadNodeListData(
   410→	cluster, subCluster string,
   411→	nodes []string,
   412→	metrics []string,
   413→	scopes []schema.MetricScope,
   414→	resolution int,
   415→	from, to time.Time,
   416→	ctx context.Context,
   417→) (map[string]schema.JobData, error) {
   418→	if metrics == nil {
   419→		for _, m := range archive.GetCluster(cluster).MetricConfig {
   420→			metrics = append(metrics, m.Name)
   421→		}
   422→	}
   423→
   424→	ms, err := GetMetricDataRepo(cluster, subCluster)
   425→	if err != nil {
   426→		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
   427→			cluster, subCluster, err.Error())
   428→		return nil, err
   429→	}
   430→
   431→	data, err := ms.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, resolution, from, to, ctx)
   432→	if err != nil {
   433→		if len(data) != 0 {
   434→			cclog.Warnf("partial error loading node list data from metric store for cluster %s, subcluster %s: %s",
   435→				cluster, subCluster, err.Error())
   436→		} else {
   437→			cclog.Warnf("failed to load node list data from metric store for cluster %s, subcluster %s: %s",
   438→				cluster, subCluster, err.Error())
   439→			return nil, err
   440→		}
   441→	}
   442→
   443→	// Generate statistics series for datasets with many series to improve visualization performance.
   444→	// Statistics are calculated as min/median/max.
   445→	const maxSeriesSize int = 8
   446→	for _, jd := range data {
   447→		for _, scopes := range jd {
   448→			for _, jm := range scopes {
   449→				if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
   450→					continue
   451→				}
   452→				jm.AddStatisticsSeries()
   453→			}
   454→		}
   455→	}
   456→
   457→	if data == nil {
   458→		return nil, fmt.Errorf("metric store for cluster '%s' does not support node list queries", cluster)
   459→	}
   460→
   461→	return data, nil
   462→}
   463→
   464→// deepCopy creates a deep copy of JobData to prevent cache corruption when modifying
   465→// archived data (e.g., during resampling). This ensures the cached archive data remains
   466→// immutable while allowing per-request transformations.
   467→func deepCopy(source schema.JobData) schema.JobData {
   468→	result := make(schema.JobData, len(source))
   469→
   470→	for metricName, scopeMap := range source {
   471→		result[metricName] = make(map[schema.MetricScope]*schema.JobMetric, len(scopeMap))
   472→
   473→		for scope, jobMetric := range scopeMap {
   474→			result[metricName][scope] = copyJobMetric(jobMetric)
   475→		}
   476→	}
   477→
   478→	return result
   479→}
   480→
   481→func copyJobMetric(src *schema.JobMetric) *schema.JobMetric {
   482→	dst := &schema.JobMetric{
   483→		Timestep: src.Timestep,
   484→		Unit:     src.Unit,
   485→		Series:   make([]schema.Series, len(src.Series)),
   486→	}
   487→
   488→	for i := range src.Series {
   489→		dst.Series[i] = copySeries(&src.Series[i])
   490→	}
   491→
   492→	if src.StatisticsSeries != nil {
   493→		dst.StatisticsSeries = copyStatisticsSeries(src.StatisticsSeries)
   494→	}
   495→
   496→	return dst
   497→}
   498→
   499→func copySeries(src *schema.Series) schema.Series {
   500→	dst := schema.Series{
   501→		Hostname:   src.Hostname,
   502→		ID:         src.ID,
   503→		Statistics: src.Statistics,
   504→		Data:       make([]schema.Float, len(src.Data)),
   505→	}
   506→
   507→	copy(dst.Data, src.Data)
   508→	return dst
   509→}
   510→
   511→func copyStatisticsSeries(src *schema.StatsSeries) *schema.StatsSeries {
   512→	dst := &schema.StatsSeries{
   513→		Min:    make([]schema.Float, len(src.Min)),
   514→		Mean:   make([]schema.Float, len(src.Mean)),
   515→		Median: make([]schema.Float, len(src.Median)),
   516→		Max:    make([]schema.Float, len(src.Max)),
   517→	}
   518→
   519→	copy(dst.Min, src.Min)
   520→	copy(dst.Mean, src.Mean)
   521→	copy(dst.Median, src.Median)
   522→	copy(dst.Max, src.Max)
   523→
   524→	if len(src.Percentiles) > 0 {
   525→		dst.Percentiles = make(map[int][]schema.Float, len(src.Percentiles))
   526→		for percentile, values := range src.Percentiles {
   527→			dst.Percentiles[percentile] = make([]schema.Float, len(values))
   528→			copy(dst.Percentiles[percentile], values)
   529→		}
   530→	}
   531→
   532→	return dst
   533→}
   534→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricdispatch provides a unified interface for loading and caching job metric data.
//
// This package serves as a central dispatcher that routes metric data requests to the appropriate
// backend based on job state. For running jobs, data is fetched from the metric store (e.g., cc-metric-store).
// For completed jobs, data is retrieved from the file-based job archive.
//
// # Key Features
//
//   - Automatic backend selection based on job state (running vs. archived)
//   - LRU cache for performance optimization (128 MB default cache size)
//   - Data resampling using Largest Triangle Three Bucket algorithm for archived data
//   - Automatic statistics series generation for jobs with many nodes
//   - Support for scoped metrics (node, socket, accelerator, core)
//
// # Cache Behavior
//
// Cached data has different TTL (time-to-live) values depending on job state:
//   - Running jobs: 2 minutes (data changes frequently)
//   - Completed jobs: 5 hours (data is static)
//
// The cache key is based on job ID, state, requested metrics, scopes, and resolution.
//
// # Usage
//
// The primary entry point is LoadData, which automatically handles both running and archived jobs:
//
//	jobData, err := metricdispatch.LoadData(job, metrics, scopes, ctx, resolution)
//	if err != nil {
//	    // Handle error
//	}
//
// For statistics only, use LoadJobStats, LoadScopedJobStats, or LoadAverages depending on the required format.
package metricdispatch

import (
	"context"
	"fmt"
	"math"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/lrucache"
	"github.com/ClusterCockpit/cc-lib/v2/resampler"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// cache is an LRU cache with 128 MB capacity for storing loaded job metric data.
// The cache reduces load on both the metric store and archive backends.
var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024)

// cacheKey generates a unique cache key for a job's metric data based on job ID, state,
// requested metrics, scopes, and resolution. Duration and StartTime are intentionally excluded
// because job.ID is more unique and the cache TTL ensures entries don't persist indefinitely.
func cacheKey(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
) string {
	return fmt.Sprintf("%d(%s):[%v],[%v]-%d",
		*job.ID, job.State, metrics, scopes, resolution)
}

// LoadData retrieves metric data for a job from the appropriate backend (memory store for running jobs,
// archive for completed jobs) and applies caching, resampling, and statistics generation as needed.
//
// For running jobs or when archive is disabled, data is fetched from the metric store.
// For completed archived jobs, data is loaded from the job archive and resampled if needed.
//
// Parameters:
//   - job: The job for which to load metric data
//   - metrics: List of metric names to load (nil loads all metrics for the cluster)
//   - scopes: Metric scopes to include (nil defaults to node scope)
//   - ctx: Context for cancellation and timeouts
//   - resolution: Target number of data points for resampling (only applies to archived data)
//
// Returns the loaded job data and any error encountered. For partial errors (some metrics failed),
// the function returns the successfully loaded data with a warning logged.
func LoadData(job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
	resolution int,
) (schema.JobData, error) {
	data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ any, ttl time.Duration, size int) {
		var jd schema.JobData
		var err error

		if job.State == schema.JobStateRunning ||
			job.MonitoringStatus == schema.MonitoringStatusRunningOrArchiving {

			ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
			if err != nil {
				cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
					job.Cluster, job.SubCluster, err.Error())
				return err, 0, 0
			}

			if scopes == nil {
				scopes = append(scopes, schema.MetricScopeNode)
			}

			if metrics == nil {
				cluster := archive.GetCluster(job.Cluster)
				for _, mc := range cluster.MetricConfig {
					metrics = append(metrics, mc.Name)
				}
			}

			jd, err = ms.LoadData(job, metrics, scopes, ctx, resolution)
			if err != nil {
				if len(jd) != 0 {
					cclog.Warnf("partial error loading metrics from store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
						job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
				} else {
					cclog.Warnf("failed to load job data from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
						job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
					return err, 0, 0
				}
			}
			size = jd.Size()
		} else {
			var jdTemp schema.JobData
			jdTemp, err = archive.GetHandle().LoadJobData(job)
			if err != nil {
				cclog.Warnf("failed to load job data from archive for job %d (user: %s, project: %s, cluster: %s-%s): %s",
					job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
				return err, 0, 0
			}

			jd = deepCopy(jdTemp)

			// Resample archived data using Largest Triangle Three Bucket algorithm to reduce data points
			// to the requested resolution, improving transfer performance and client-side rendering.
			for _, v := range jd {
				for _, v_ := range v {
					timestep := int64(0)
					for i := 0; i < len(v_.Series); i += 1 {
						v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))
						if err != nil {
							return err, 0, 0
						}
					}
					v_.Timestep = int(timestep)
				}
			}

			// Filter job data to only include requested metrics and scopes, avoiding unnecessary data transfer.
			if metrics != nil || scopes != nil {
				if metrics == nil {
					metrics = make([]string, 0, len(jd))
					for k := range jd {
						metrics = append(metrics, k)
					}
				}

				res := schema.JobData{}
				for _, metric := range metrics {
					if perscope, ok := jd[metric]; ok {
						if len(perscope) > 1 {
							subset := make(map[schema.MetricScope]*schema.JobMetric)
							for _, scope := range scopes {
								if jm, ok := perscope[scope]; ok {
									subset[scope] = jm
								}
							}

							if len(subset) > 0 {
								perscope = subset
							}
						}

						res[metric] = perscope
					}
				}
				jd = res
			}
			size = jd.Size()
		}

		ttl = 5 * time.Hour
		if job.State == schema.JobStateRunning {
			ttl = 2 * time.Minute
		}

		// Generate statistics series for jobs with many nodes to enable min/median/max graphs
		// instead of overwhelming the UI with individual node lines. Note that newly calculated
		// statistics use min/median/max, while archived statistics may use min/mean/max.
		const maxSeriesSize int = 8
		for _, scopes := range jd {
			for _, jm := range scopes {
				if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
					continue
				}

				jm.AddStatisticsSeries()
			}
		}

		nodeScopeRequested := false
		for _, scope := range scopes {
			if scope == schema.MetricScopeNode {
				nodeScopeRequested = true
			}
		}

		if nodeScopeRequested {
			jd.AddNodeScope("flops_any")
			jd.AddNodeScope("mem_bw")
		}

		// Round Resulting Stat Values
		jd.RoundMetricStats()

		return jd, ttl, size
	})

	if err, ok := data.(error); ok {
		cclog.Errorf("error in cached dataset for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	return data.(schema.JobData), nil
}

// LoadAverages computes average values for the specified metrics across all nodes of a job.
// For running jobs, it loads statistics from the metric store. For completed jobs, it uses
// the pre-calculated averages from the job archive. The results are appended to the data slice.
func LoadAverages(
	job *schema.Job,
	metrics []string,
	data [][]schema.Float,
	ctx context.Context,
) error {
	if job.State != schema.JobStateRunning {
		return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here?
	}

	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			job.Cluster, job.SubCluster, err.Error())
		return err
	}

	stats, err := ms.LoadStats(job, metrics, ctx)
	if err != nil {
		cclog.Warnf("failed to load statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
		return err
	}

	for i, m := range metrics {
		nodes, ok := stats[m]
		if !ok {
			data[i] = append(data[i], schema.NaN)
			continue
		}

		sum := 0.0
		for _, node := range nodes {
			sum += node.Avg
		}
		data[i] = append(data[i], schema.Float(sum))
	}

	return nil
}

// LoadScopedJobStats retrieves job statistics organized by metric scope (node, socket, core, accelerator).
// For running jobs, statistics are computed from the metric store. For completed jobs, pre-calculated
// statistics are loaded from the job archive.
func LoadScopedJobStats(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
) (schema.ScopedJobStats, error) {
	if job.State != schema.JobStateRunning {
		return archive.LoadScopedStatsFromArchive(job, metrics, scopes)
	}

	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			job.Cluster, job.SubCluster, err.Error())
		return nil, err
	}

	scopedStats, err := ms.LoadScopedStats(job, metrics, scopes, ctx)
	if err != nil {
		cclog.Warnf("failed to load scoped statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
		return nil, err
	}

	// Round Resulting Stat Values
	scopedStats.RoundScopedMetricStats()

	return scopedStats, nil
}

// LoadJobStats retrieves aggregated statistics (min/avg/max) for each requested metric across all job nodes.
// For running jobs, statistics are computed from the metric store. For completed jobs, pre-calculated
// statistics are loaded from the job archive.
func LoadJobStats(
	job *schema.Job,
	metrics []string,
	ctx context.Context,
) (map[string]schema.MetricStatistics, error) {
	if job.State != schema.JobStateRunning {
		return archive.LoadStatsFromArchive(job, metrics)
	}

	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			job.Cluster, job.SubCluster, err.Error())
		return nil, err
	}

	data := make(map[string]schema.MetricStatistics, len(metrics))

	stats, err := ms.LoadStats(job, metrics, ctx)
	if err != nil {
		cclog.Warnf("failed to load statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
		return data, err
	}

	for _, m := range metrics {
		sum, avg, min, max := 0.0, 0.0, 0.0, 0.0
		nodes, ok := stats[m]
		if !ok {
			data[m] = schema.MetricStatistics{Min: min, Avg: avg, Max: max}
			continue
		}

		for _, node := range nodes {
			sum += node.Avg
			min = math.Min(min, node.Min)
			max = math.Max(max, node.Max)
		}

		data[m] = schema.MetricStatistics{
			Avg: (math.Round((sum/float64(job.NumNodes))*100) / 100),
			Min: (math.Round(min*100) / 100),
			Max: (math.Round(max*100) / 100),
		}
	}

	return data, nil
}

// LoadNodeData retrieves metric data for specific nodes in a cluster within a time range.
// This is used for node monitoring views and system status pages. Data is always fetched from
// the metric store (not the archive) since it's for current/recent node status monitoring.
//
// Returns a nested map structure: node -> metric -> scoped data.
// FIXME: Add support for subcluster specific cc-metric-stores
func LoadNodeData(
	cluster string,
	metrics, nodes []string,
	scopes []schema.MetricScope,
	from, to time.Time,
	ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
	if metrics == nil {
		for _, m := range archive.GetCluster(cluster).MetricConfig {
			metrics = append(metrics, m.Name)
		}
	}

	ms, err := GetMetricDataRepo(cluster, "")
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s: %s",
			cluster, err.Error())
		return nil, err
	}

	data, err := ms.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
	if err != nil {
		if len(data) != 0 {
			cclog.Warnf("partial error loading node data from metric store for cluster %s: %s", cluster, err.Error())
		} else {
			cclog.Warnf("failed to load node data from metric store for cluster %s: %s", cluster, err.Error())
			return nil, err
		}
	}

	if data == nil {
		return nil, fmt.Errorf("metric store for cluster '%s' does not support node data queries", cluster)
	}

	return data, nil
}

// LoadNodeListData retrieves time-series metric data for multiple nodes within a time range,
// with optional resampling and automatic statistics generation for large datasets.
// This is used for comparing multiple nodes or displaying node status over time.
//
// Returns a map of node names to their job-like metric data structures.
func LoadNodeListData(
	cluster, subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	from, to time.Time,
	ctx context.Context,
) (map[string]schema.JobData, error) {
	if metrics == nil {
		for _, m := range archive.GetCluster(cluster).MetricConfig {
			metrics = append(metrics, m.Name)
		}
	}

	ms, err := GetMetricDataRepo(cluster, subCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			cluster, subCluster, err.Error())
		return nil, err
	}

	data, err := ms.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, resolution, from, to, ctx)
	if err != nil {
		if len(data) != 0 {
			cclog.Warnf("partial error loading node list data from metric store for cluster %s, subcluster %s: %s",
				cluster, subCluster, err.Error())
		} else {
			cclog.Warnf("failed to load node list data from metric store for cluster %s, subcluster %s: %s",
				cluster, subCluster, err.Error())
			return nil, err
		}
	}

	// Generate statistics series for datasets with many series to improve visualization performance.
	// Statistics are calculated as min/median/max.
	const maxSeriesSize int = 8
	for _, jd := range data {
		for _, scopes := range jd {
			for _, jm := range scopes {
				if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
					continue
				}
				jm.AddStatisticsSeries()
			}
		}
	}

	if data == nil {
		return nil, fmt.Errorf("metric store for cluster '%s' does not support node list queries", cluster)
	}

	return data, nil
}

// deepCopy creates a deep copy of JobData to prevent cache corruption when modifying
// archived data (e.g., during resampling). This ensures the cached archive data remains
// immutable while allowing per-request transformations.
func deepCopy(source schema.JobData) schema.JobData {
	result := make(schema.JobData, len(source))

	for metricName, scopeMap := range source {
		result[metricName] = make(map[schema.MetricScope]*schema.JobMetric, len(scopeMap))

		for scope, jobMetric := range scopeMap {
			result[metricName][scope] = copyJobMetric(jobMetric)
		}
	}

	return result
}

func copyJobMetric(src *schema.JobMetric) *schema.JobMetric {
	dst := &schema.JobMetric{
		Timestep: src.Timestep,
		Unit:     src.Unit,
		Series:   make([]schema.Series, len(src.Series)),
	}

	for i := range src.Series {
		dst.Series[i] = copySeries(&src.Series[i])
	}

	if src.StatisticsSeries != nil {
		dst.StatisticsSeries = copyStatisticsSeries(src.StatisticsSeries)
	}

	return dst
}

func copySeries(src *schema.Series) schema.Series {
	dst := schema.Series{
		Hostname:   src.Hostname,
		ID:         src.ID,
		Statistics: src.Statistics,
		Data:       make([]schema.Float, len(src.Data)),
	}

	copy(dst.Data, src.Data)
	return dst
}

func copyStatisticsSeries(src *schema.StatsSeries) *schema.StatsSeries {
	dst := &schema.StatsSeries{
		Min:    make([]schema.Float, len(src.Min)),
		Mean:   make([]schema.Float, len(src.Mean)),
		Median: make([]schema.Float, len(src.Median)),
		Max:    make([]schema.Float, len(src.Max)),
	}

	copy(dst.Min, src.Min)
	copy(dst.Mean, src.Mean)
	copy(dst.Median, src.Median)
	copy(dst.Max, src.Max)

	if len(src.Percentiles) > 0 {
		dst.Percentiles = make(map[int][]schema.Float, len(src.Percentiles))
		for percentile, values := range src.Percentiles {
			dst.Percentiles[percentile] = make([]schema.Float, len(values))
			copy(dst.Percentiles[percentile], values)
		}
	}

	return dst
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides an efficient in-memory time-series metric storage system
     7→// with support for hierarchical data organization, checkpointing, and archiving.
     8→//
     9→// The package organizes metrics in a tree structure (cluster → host → component) and
    10→// provides concurrent read/write access to metric data with configurable aggregation strategies.
    11→// Background goroutines handle periodic checkpointing (JSON or Avro format), archiving old data,
    12→// and enforcing retention policies.
    13→//
    14→// Key features:
    15→//   - In-memory metric storage with configurable retention
    16→//   - Hierarchical data organization (selectors)
    17→//   - Concurrent checkpoint/archive workers
    18→//   - Support for sum and average aggregation
    19→//   - NATS integration for metric ingestion
    20→package metricstore
    21→
    22→import (
    23→	"bytes"
    24→	"context"
    25→	"encoding/json"
    26→	"errors"
    27→	"fmt"
    28→	"runtime"
    29→	"runtime/debug"
    30→	"slices"
    31→	"sync"
    32→	"time"
    33→
    34→	"github.com/ClusterCockpit/cc-backend/internal/config"
    35→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    36→	"github.com/ClusterCockpit/cc-lib/v2/resampler"
    37→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    38→	"github.com/ClusterCockpit/cc-lib/v2/util"
    39→)
    40→
    41→// GlobalState holds the global state for the metric store with thread-safe access.
    42→type GlobalState struct {
    43→	mu                sync.RWMutex
    44→	lastRetentionTime int64
    45→	selectorsExcluded bool
    46→}
    47→
    48→var (
    49→	singleton  sync.Once
    50→	msInstance *MemoryStore
    51→	// shutdownFunc stores the context cancellation function created in Init
    52→	// and is called during Shutdown to cancel all background goroutines
    53→	shutdownFunc   context.CancelFunc
    54→	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
    55→	// Create a global instance
    56→	state = &GlobalState{}
    57→)
    58→
    59→// NodeProvider provides information about nodes currently in use by running jobs.
    60→//
    61→// This interface allows metricstore to query job information without directly
    62→// depending on the repository package, breaking the import cycle.
    63→//
    64→// Implementations should return nodes that are actively processing jobs started
    65→// before the given timestamp. These nodes will be excluded from retention-based
    66→// garbage collection to prevent data loss for jobs that are still running or
    67→// recently completed.
    68→type NodeProvider interface {
    69→	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
    70→	// that are currently in use by jobs that started before the given timestamp.
    71→	//
    72→	// Parameters:
    73→	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
    74→	//
    75→	// Returns:
    76→	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
    77→	//   - Error if the query fails
    78→	GetUsedNodes(ts int64) (map[string][]string, error)
    79→}
    80→
    81→// Metric represents a single metric data point to be written to the store.
    82→type Metric struct {
    83→	Name  string
    84→	Value schema.Float
    85→	// MetricConfig contains frequency and aggregation settings for this metric.
    86→	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
    87→	MetricConfig MetricConfig
    88→}
    89→
    90→// MemoryStore is the main in-memory time-series metric storage implementation.
    91→//
    92→// It organizes metrics in a hierarchical tree structure where each level represents
    93→// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
    94→// store multiple metrics as time-series buffers.
    95→//
    96→// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
    97→// All public methods are safe for concurrent use.
    98→type MemoryStore struct {
    99→	Metrics      map[string]MetricConfig
   100→	root         Level
   101→	nodeProvider NodeProvider
   102→}
   103→
   104→// Init initializes the metric store from configuration and starts background workers.
   105→//
   106→// This function must be called exactly once before any other metricstore operations.
   107→// It performs the following initialization steps:
   108→//  1. Validates and decodes the metric store configuration
   109→//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
   110→//  3. Loads metric configurations from all registered clusters
   111→//  4. Restores checkpoints within the retention window
   112→//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
   113→//  6. Optionally subscribes to NATS for real-time metric ingestion
   114→//
   115→// Parameters:
   116→//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig)
   117→//   - wg: WaitGroup that will be incremented for each background goroutine started
   118→//
   119→// The function will call cclog.Fatal on critical errors during initialization.
   120→// Use Shutdown() to cleanly stop all background workers started by Init().
   121→//
   122→// Note: Signal handling must be implemented by the caller. Call Shutdown() when
   123→// receiving termination signals to ensure checkpoint data is persisted.
   124→func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
   125→	startupTime := time.Now()
   126→
   127→	if rawConfig != nil {
   128→		config.Validate(configSchema, rawConfig)
   129→		dec := json.NewDecoder(bytes.NewReader(rawConfig))
   130→		dec.DisallowUnknownFields()
   131→		if err := dec.Decode(&Keys); err != nil {
   132→			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
   133→		}
   134→	}
   135→
   136→	// Set NumWorkers from config or use default
   137→	if Keys.NumWorkers <= 0 {
   138→		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
   139→	}
   140→	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
   141→
   142→	// Pass the config.MetricStoreKeys
   143→	InitMetrics(metrics)
   144→
   145→	ms := GetMemoryStore()
   146→
   147→	d, err := time.ParseDuration(Keys.RetentionInMemory)
   148→	if err != nil {
   149→		cclog.Fatal(err)
   150→	}
   151→
   152→	restoreFrom := startupTime.Add(-d)
   153→	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
   154→	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
   155→	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
   156→	if err != nil {
   157→		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
   158→	} else {
   159→		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
   160→	}
   161→
   162→	// Try to use less memory by forcing a GC run here and then
   163→	// lowering the target percentage. The default of 100 means
   164→	// that only once the ratio of new allocations execeds the
   165→	// previously active heap, a GC is triggered.
   166→	// Forcing a GC here will set the "previously active heap"
   167→	// to a minumum.
   168→	// runtime.GC()
   169→
   170→	ctx, shutdown := context.WithCancel(context.Background())
   171→
   172→	Retention(wg, ctx)
   173→	Checkpointing(wg, ctx)
   174→	CleanUp(wg, ctx)
   175→	WALStaging(wg, ctx)
   176→	MemoryUsageTracker(wg, ctx)
   177→
   178→	// Note: Signal handling has been removed from this function.
   179→	// The caller is responsible for handling shutdown signals and calling
   180→	// the shutdown() function when appropriate.
   181→	// Store the shutdown function for later use by Shutdown()
   182→	shutdownFuncMu.Lock()
   183→	shutdownFunc = shutdown
   184→	shutdownFuncMu.Unlock()
   185→
   186→	if Keys.Subscriptions != nil {
   187→		wg.Go(func() {
   188→			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
   189→				cclog.Fatal(err)
   190→			}
   191→		})
   192→	}
   193→}
   194→
   195→// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
   196→//
   197→// This function must be called before GetMemoryStore() and can only be called once due to
   198→// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
   199→//
   200→// Parameters:
   201→//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
   202→//
   203→// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
   204→//
   205→// After this call, the global msInstance is ready for use via GetMemoryStore().
   206→func InitMetrics(metrics map[string]MetricConfig) {
   207→	singleton.Do(func() {
   208→		offset := 0
   209→		for key, cfg := range metrics {
   210→			if cfg.Frequency == 0 {
   211→				panic("[METRICSTORE]> invalid frequency")
   212→			}
   213→
   214→			metrics[key] = MetricConfig{
   215→				Frequency:   cfg.Frequency,
   216→				Aggregation: cfg.Aggregation,
   217→				offset:      offset,
   218→			}
   219→			offset += 1
   220→		}
   221→
   222→		msInstance = &MemoryStore{
   223→			root: Level{
   224→				metrics:  make([]*buffer, len(metrics)),
   225→				children: make(map[string]*Level),
   226→			},
   227→			Metrics: metrics,
   228→		}
   229→	})
   230→}
   231→
   232→// GetMemoryStore returns the singleton MemoryStore instance.
   233→//
   234→// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
   235→//
   236→// This function is safe for concurrent use after initialization.
   237→func GetMemoryStore() *MemoryStore {
   238→	if msInstance == nil {
   239→		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
   240→	}
   241→
   242→	return msInstance
   243→}
   244→
   245→func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
   246→	if metric, ok := ms.Metrics[metricName]; ok {
   247→		return metric.Frequency, nil
   248→	}
   249→	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
   250→}
   251→
   252→// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
   253→// This must be called during initialization to provide job state information
   254→// for selective buffer retention during Free operations.
   255→// If not set, the Free function will fall back to freeing all buffers.
   256→func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
   257→	ms.nodeProvider = provider
   258→}
   259→
   260→// Shutdown performs a graceful shutdown of the metric store.
   261→//
   262→// This function cancels all background goroutines started by Init() and writes
   263→// a final checkpoint to disk before returning. It should be called when the
   264→// application receives a termination signal.
   265→//
   266→// The function will:
   267→//  1. Cancel the context to stop all background workers
   268→//  2. Close the WAL messages channel if using WAL format
   269→//  3. Write a final checkpoint to preserve in-memory data
   270→//  4. Log any errors encountered during shutdown
   271→//
   272→// Note: This function blocks until the final checkpoint is written.
   273→func Shutdown() {
   274→	shutdownFuncMu.Lock()
   275→	defer shutdownFuncMu.Unlock()
   276→	if shutdownFunc != nil {
   277→		shutdownFunc()
   278→	}
   279→
   280→	if Keys.Checkpoints.FileFormat == "wal" {
   281→		for _, ch := range walShardChs {
   282→			close(ch)
   283→		}
   284→	}
   285→
   286→	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
   287→	var files int
   288→	var err error
   289→
   290→	ms := GetMemoryStore()
   291→
   292→	lastCheckpointMu.Lock()
   293→	from := lastCheckpoint
   294→	lastCheckpointMu.Unlock()
   295→
   296→	if Keys.Checkpoints.FileFormat == "wal" {
   297→		var hostDirs []string
   298→		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   299→		if err == nil {
   300→			RotateWALFilesAfterShutdown(hostDirs)
   301→		}
   302→	} else {
   303→		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
   304→	}
   305→
   306→	if err != nil {
   307→		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
   308→	}
   309→	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
   310→}
   311→
   312→// Retention starts a background goroutine that periodically frees old metric data.
   313→//
   314→// This worker runs at half the retention interval and calls Free() to remove buffers
   315→// older than the configured retention time. It respects the NodeProvider to preserve
   316→// data for nodes with active jobs.
   317→//
   318→// Parameters:
   319→//   - wg: WaitGroup to signal completion when context is cancelled
   320→//   - ctx: Context for cancellation signal
   321→//
   322→// The goroutine exits when ctx is cancelled.
   323→func Retention(wg *sync.WaitGroup, ctx context.Context) {
   324→	ms := GetMemoryStore()
   325→
   326→	wg.Go(func() {
   327→		d, err := time.ParseDuration(Keys.RetentionInMemory)
   328→		if err != nil {
   329→			cclog.Fatal(err)
   330→		}
   331→		if d <= 0 {
   332→			return
   333→		}
   334→
   335→		tickInterval := d / 2
   336→		if tickInterval <= 0 {
   337→			return
   338→		}
   339→		ticker := time.NewTicker(tickInterval)
   340→		defer ticker.Stop()
   341→
   342→		for {
   343→			select {
   344→			case <-ctx.Done():
   345→				return
   346→			case <-ticker.C:
   347→				state.mu.Lock()
   348→
   349→				t := time.Now().Add(-d)
   350→
   351→				state.lastRetentionTime = t.Unix()
   352→
   353→				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
   354→
   355→				freed, err := Free(ms, t)
   356→				if err != nil {
   357→					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
   358→				} else {
   359→					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
   360→				}
   361→
   362→				state.mu.Unlock()
   363→
   364→				// Clean up the buffer pool
   365→				bufferPool.Clean(state.lastRetentionTime)
   366→			}
   367→		}
   368→	})
   369→}
   370→
   371→// MemoryUsageTracker starts a background goroutine that monitors memory usage.
   372→//
   373→// This worker checks actual process memory usage (via runtime.MemStats) periodically
   374→// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
   375→// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
   376→// performance issues.
   377→//
   378→// The tracker logs both actual memory usage (heap allocated) and metric data size for
   379→// visibility into memory overhead from Go runtime structures and allocations.
   380→//
   381→// Parameters:
   382→//   - wg: WaitGroup to signal completion when context is cancelled
   383→//   - ctx: Context for cancellation signal
   384→//
   385→// The goroutine exits when ctx is cancelled.
   386→func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
   387→	ms := GetMemoryStore()
   388→
   389→	wg.Go(func() {
   390→		normalInterval := DefaultMemoryUsageTrackerInterval
   391→		fastInterval := 30 * time.Second
   392→
   393→		if normalInterval <= 0 {
   394→			return
   395→		}
   396→
   397→		ticker := time.NewTicker(normalInterval)
   398→		defer ticker.Stop()
   399→		currentInterval := normalInterval
   400→
   401→		for {
   402→			select {
   403→			case <-ctx.Done():
   404→				return
   405→			case <-ticker.C:
   406→				var mem runtime.MemStats
   407→				runtime.ReadMemStats(&mem)
   408→				actualMemoryGB := float64(mem.Alloc) / 1e9
   409→				metricDataGB := ms.SizeInGB()
   410→				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
   411→
   412→				freedExcluded := 0
   413→				freedEmergency := 0
   414→				var err error
   415→
   416→				state.mu.RLock()
   417→				lastRetention := state.lastRetentionTime
   418→				selectorsExcluded := state.selectorsExcluded
   419→				state.mu.RUnlock()
   420→
   421→				if lastRetention != 0 && selectorsExcluded {
   422→					freedExcluded, err = ms.Free(nil, lastRetention)
   423→					if err != nil {
   424→						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
   425→					}
   426→
   427→					if freedExcluded > 0 {
   428→						debug.FreeOSMemory()
   429→						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
   430→					}
   431→				}
   432→
   433→				runtime.ReadMemStats(&mem)
   434→				actualMemoryGB = float64(mem.Alloc) / 1e9
   435→
   436→				if actualMemoryGB > float64(Keys.MemoryCap) {
   437→					cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)
   438→
   439→					// Use progressive time-based Free with increasing threshold
   440→					// instead of ForceFree loop — fewer tree traversals, more effective
   441→					d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
   442→					if parseErr != nil {
   443→						cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
   444→					} else {
   445→						thresholds := []float64{0.75, 0.5, 0.25}
   446→						for _, fraction := range thresholds {
   447→							threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
   448→							freed, freeErr := ms.Free(nil, threshold.Unix())
   449→							if freeErr != nil {
   450→								cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
   451→							}
   452→							freedEmergency += freed
   453→
   454→							bufferPool.Clear()
   455→							runtime.GC()
   456→							runtime.ReadMemStats(&mem)
   457→							actualMemoryGB = float64(mem.Alloc) / 1e9
   458→
   459→							if actualMemoryGB < float64(Keys.MemoryCap) {
   460→								break
   461→							}
   462→						}
   463→					}
   464→
   465→					bufferPool.Clear()
   466→					debug.FreeOSMemory()
   467→
   468→					runtime.ReadMemStats(&mem)
   469→					actualMemoryGB = float64(mem.Alloc) / 1e9
   470→
   471→					if actualMemoryGB >= float64(Keys.MemoryCap) {
   472→						cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
   473→					} else {
   474→						cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
   475→					}
   476→				}
   477→
   478→				// Adaptive ticker: check more frequently when memory is high
   479→				memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
   480→				if memoryRatio > 0.8 && currentInterval != fastInterval {
   481→					ticker.Reset(fastInterval)
   482→					currentInterval = fastInterval
   483→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
   484→				} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
   485→					ticker.Reset(normalInterval)
   486→					currentInterval = normalInterval
   487→					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
   488→				}
   489→			}
   490→		}
   491→	})
   492→}
   493→
   494→// Free removes metric data older than the given time while preserving data for active nodes.
   495→//
   496→// This function implements intelligent retention by consulting the NodeProvider (if configured)
   497→// to determine which nodes are currently in use by running jobs. Data for these nodes is
   498→// preserved even if older than the retention time.
   499→//
   500→// Parameters:
   501→//   - ms: The MemoryStore instance
   502→//   - t: Time threshold - buffers with data older than this will be freed
   503→//
   504→// Returns:
   505→//   - Number of buffers freed
   506→//   - Error if NodeProvider query fails
   507→//
   508→// Behavior:
   509→//   - If no NodeProvider is set: frees all buffers older than t
   510→//   - If NodeProvider returns empty map: frees all buffers older than t
   511→//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
   512→func Free(ms *MemoryStore, t time.Time) (int, error) {
   513→	// If no NodeProvider is configured, free all buffers older than t
   514→	if ms.nodeProvider == nil {
   515→		return ms.Free(nil, t.Unix())
   516→	}
   517→
   518→	excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
   519→	if err != nil {
   520→		return 0, err
   521→	}
   522→
   523→	switch lenMap := len(excludeSelectors); lenMap {
   524→
   525→	// If the length of the map returned by GetUsedNodes() is 0,
   526→	// then use default Free method with nil selector
   527→	case 0:
   528→		state.selectorsExcluded = false
   529→		return ms.Free(nil, t.Unix())
   530→
   531→	// Else formulate selectors, exclude those from the map
   532→	// and free the rest of the selectors
   533→	default:
   534→		state.selectorsExcluded = true
   535→		selectors := GetSelectors(ms, excludeSelectors)
   536→		return FreeSelected(ms, selectors, t)
   537→	}
   538→}
   539→
   540→// FreeSelected frees buffers for specific selectors while preserving others.
   541→//
   542→// This function is used when we want to retain some specific nodes beyond the retention time.
   543→// It iterates through the provided selectors and frees their associated buffers.
   544→//
   545→// Parameters:
   546→//   - ms: The MemoryStore instance
   547→//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
   548→//   - t: Time threshold for freeing buffers
   549→//
   550→// Returns the total number of buffers freed and any error encountered.
   551→func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
   552→	freed := 0
   553→
   554→	for _, selector := range selectors {
   555→
   556→		freedBuffers, err := ms.Free(selector, t.Unix())
   557→		if err != nil {
   558→			cclog.Errorf("error while freeing selected buffers: %#v", err)
   559→		}
   560→		freed += freedBuffers
   561→
   562→	}
   563→
   564→	return freed, nil
   565→}
   566→
   567→// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
   568→//
   569→// This function generates a list of selectors whose buffers should be freed by excluding
   570→// selectors that correspond to nodes currently in use by running jobs.
   571→//
   572→// Parameters:
   573→//   - ms: The MemoryStore instance
   574→//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
   575→//
   576→// Returns a list of selectors ([]string paths) that can be safely freed.
   577→//
   578→// Example:
   579→//
   580→//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
   581→//	and excludeSelectors contains {"emmy": ["node001"]},
   582→//	then only [["emmy", "node002"]] is returned.
   583→func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
   584→	allSelectors := ms.GetPaths(2)
   585→
   586→	filteredSelectors := make([][]string, 0, len(allSelectors))
   587→
   588→	for _, path := range allSelectors {
   589→		if len(path) < 2 {
   590→			continue
   591→		}
   592→
   593→		key := path[0]   // The "Key" (Level 1)
   594→		value := path[1] // The "Value" (Level 2)
   595→
   596→		exclude := false
   597→
   598→		// Check if the key exists in our exclusion map
   599→		if excludedValues, exists := excludeSelectors[key]; exists {
   600→			// The key exists, now check if the specific value is in the exclusion list
   601→			if slices.Contains(excludedValues, value) {
   602→				exclude = true
   603→			}
   604→		}
   605→
   606→		if !exclude {
   607→			filteredSelectors = append(filteredSelectors, path)
   608→		}
   609→	}
   610→
   611→	return filteredSelectors
   612→}
   613→
   614→// GetPaths returns a list of lists (paths) to the specified depth.
   615→func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
   616→	var results [][]string
   617→
   618→	// Start recursion. Initial path is empty.
   619→	// We treat Root as depth 0.
   620→	ms.root.collectPaths(0, targetDepth, []string{}, &results)
   621→
   622→	return results
   623→}
   624→
   625→// Write all values in `metrics` to the level specified by `selector` for time `ts`.
   626→// Look at `findLevelOrCreate` for how selectors work.
   627→func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
   628→	var ok bool
   629→	for i, metric := range metrics {
   630→		if metric.MetricConfig.Frequency == 0 {
   631→			metric.MetricConfig, ok = m.Metrics[metric.Name]
   632→			if !ok {
   633→				cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
   634→				metric.MetricConfig.Frequency = 0
   635→			}
   636→			metrics[i] = metric
   637→		}
   638→	}
   639→
   640→	return m.WriteToLevel(&m.root, selector, ts, metrics)
   641→}
   642→
   643→func (m *MemoryStore) GetLevel(selector []string) *Level {
   644→	return m.root.findLevelOrCreate(selector, len(m.Metrics))
   645→}
   646→
   647→// WriteToLevel assumes that `minfo` in `metrics` is filled in
   648→func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
   649→	l = l.findLevelOrCreate(selector, len(m.Metrics))
   650→	l.lock.Lock()
   651→	defer l.lock.Unlock()
   652→
   653→	for _, metric := range metrics {
   654→		if metric.MetricConfig.Frequency == 0 {
   655→			continue
   656→		}
   657→
   658→		b := l.metrics[metric.MetricConfig.offset]
   659→		if b == nil {
   660→			// First write to this metric and level
   661→			b = newBuffer(ts, metric.MetricConfig.Frequency)
   662→			l.metrics[metric.MetricConfig.offset] = b
   663→		}
   664→
   665→		nb, err := b.write(ts, metric.Value)
   666→		if err != nil {
   667→			return err
   668→		}
   669→
   670→		// Last write created a new buffer...
   671→		if b != nb {
   672→			l.metrics[metric.MetricConfig.offset] = nb
   673→		}
   674→	}
   675→	return nil
   676→}
   677→
   678→// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
   679→// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
   680→// The second and third return value are the actual from/to for the data. Those can be different from
   681→// the range asked for if no data was available.
   682→func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
   683→	if from > to {
   684→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
   685→	}
   686→
   687→	minfo, ok := m.Metrics[metric]
   688→	if !ok {
   689→		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
   690→	}
   691→
   692→	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)
   693→
   694→	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
   695→		cdata, cfrom, cto, err := b.read(from, to, data)
   696→		if err != nil {
   697→			return err
   698→		}
   699→
   700→		if n == 0 {
   701→			from, to = cfrom, cto
   702→		} else if from != cfrom || to != cto || len(data) != len(cdata) {
   703→			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
   704→			if missingfront != 0 {
   705→				return ErrDataDoesNotAlign
   706→			}
   707→
   708→			newlen := len(cdata) - missingback
   709→			if newlen < 1 {
   710→				return ErrDataDoesNotAlign
   711→			}
   712→			cdata = cdata[0:newlen]
   713→			if len(cdata) != len(data) {
   714→				return ErrDataDoesNotAlign
   715→			}
   716→
   717→			from, to = cfrom, cto
   718→		}
   719→
   720→		data = cdata
   721→		n += 1
   722→		return nil
   723→	})
   724→
   725→	if err != nil {
   726→		return nil, 0, 0, 0, err
   727→	} else if n == 0 {
   728→		return nil, 0, 0, 0, ErrNoHostOrMetric
   729→	} else if n > 1 {
   730→		if minfo.Aggregation == AvgAggregation {
   731→			normalize := 1. / schema.Float(n)
   732→			for i := 0; i < len(data); i++ {
   733→				data[i] *= normalize
   734→			}
   735→		} else if minfo.Aggregation != SumAggregation {
   736→			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
   737→		}
   738→	}
   739→
   740→	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
   741→	if err != nil {
   742→		return nil, 0, 0, 0, err
   743→	}
   744→
   745→	return data, from, to, resolution, nil
   746→}
   747→
   748→// Free releases all buffers for the selected level and all its children that
   749→// contain only values older than `t`.
   750→func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
   751→	return m.GetLevel(selector).free(t)
   752→}
   753→
   754→// ForceFree unconditionally removes the oldest buffer from each metric chain.
   755→func (m *MemoryStore) ForceFree() (int, error) {
   756→	return m.GetLevel(nil).forceFree()
   757→}
   758→
   759→func (m *MemoryStore) FreeAll() error {
   760→	m.root.lock.Lock()
   761→	defer m.root.lock.Unlock()
   762→
   763→	for k := range m.root.children {
   764→		delete(m.root.children, k)
   765→	}
   766→
   767→	return nil
   768→}
   769→
   770→func (m *MemoryStore) SizeInBytes() int64 {
   771→	return m.root.sizeInBytes()
   772→}
   773→
   774→func (m *MemoryStore) SizeInGB() float64 {
   775→	return float64(m.root.sizeInBytes()) / 1e9
   776→}
   777→
   778→// ListChildren , given a selector, returns a list of all children of the level
   779→// selected.
   780→func (m *MemoryStore) ListChildren(selector []string) []string {
   781→	lvl := &m.root
   782→	for lvl != nil && len(selector) != 0 {
   783→		lvl.lock.RLock()
   784→		next := lvl.children[selector[0]]
   785→		lvl.lock.RUnlock()
   786→		lvl = next
   787→		selector = selector[1:]
   788→	}
   789→
   790→	if lvl == nil {
   791→		return nil
   792→	}
   793→
   794→	lvl.lock.RLock()
   795→	defer lvl.lock.RUnlock()
   796→
   797→	children := make([]string, 0, len(lvl.children))
   798→	for child := range lvl.children {
   799→		children = append(children, child)
   800→	}
   801→
   802→	return children
   803→}
   804→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstore provides an efficient in-memory time-series metric storage system
// with support for hierarchical data organization, checkpointing, and archiving.
//
// The package organizes metrics in a tree structure (cluster → host → component) and
// provides concurrent read/write access to metric data with configurable aggregation strategies.
// Background goroutines handle periodic checkpointing (JSON or Avro format), archiving old data,
// and enforcing retention policies.
//
// Key features:
//   - In-memory metric storage with configurable retention
//   - Hierarchical data organization (selectors)
//   - Concurrent checkpoint/archive workers
//   - Support for sum and average aggregation
//   - NATS integration for metric ingestion
package metricstore

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"runtime"
	"runtime/debug"
	"slices"
	"sync"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/config"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/resampler"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/ClusterCockpit/cc-lib/v2/util"
)

// GlobalState holds the global state for the metric store with thread-safe access.
type GlobalState struct {
	mu                sync.RWMutex
	lastRetentionTime int64
	selectorsExcluded bool
}

var (
	singleton  sync.Once
	msInstance *MemoryStore
	// shutdownFunc stores the context cancellation function created in Init
	// and is called during Shutdown to cancel all background goroutines
	shutdownFunc   context.CancelFunc
	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
	// Create a global instance
	state = &GlobalState{}
)

// NodeProvider provides information about nodes currently in use by running jobs.
//
// This interface allows metricstore to query job information without directly
// depending on the repository package, breaking the import cycle.
//
// Implementations should return nodes that are actively processing jobs started
// before the given timestamp. These nodes will be excluded from retention-based
// garbage collection to prevent data loss for jobs that are still running or
// recently completed.
type NodeProvider interface {
	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
	// that are currently in use by jobs that started before the given timestamp.
	//
	// Parameters:
	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
	//
	// Returns:
	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
	//   - Error if the query fails
	GetUsedNodes(ts int64) (map[string][]string, error)
}

// Metric represents a single metric data point to be written to the store.
type Metric struct {
	Name  string
	Value schema.Float
	// MetricConfig contains frequency and aggregation settings for this metric.
	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
	MetricConfig MetricConfig
}

// MemoryStore is the main in-memory time-series metric storage implementation.
//
// It organizes metrics in a hierarchical tree structure where each level represents
// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
// store multiple metrics as time-series buffers.
//
// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
// All public methods are safe for concurrent use.
type MemoryStore struct {
	Metrics      map[string]MetricConfig
	root         Level
	nodeProvider NodeProvider
}

// Init initializes the metric store from configuration and starts background workers.
//
// This function must be called exactly once before any other metricstore operations.
// It performs the following initialization steps:
//  1. Validates and decodes the metric store configuration
//  2. Configures worker pool size (defaults to NumCPU/2+1, max 10)
//  3. Loads metric configurations from all registered clusters
//  4. Restores checkpoints within the retention window
//  5. Starts background workers for retention, checkpointing, archiving, and monitoring
//  6. Optionally subscribes to NATS for real-time metric ingestion
//
// Parameters:
//   - rawConfig: JSON configuration for the metric store (see MetricStoreConfig)
//   - wg: WaitGroup that will be incremented for each background goroutine started
//
// The function will call cclog.Fatal on critical errors during initialization.
// Use Shutdown() to cleanly stop all background workers started by Init().
//
// Note: Signal handling must be implemented by the caller. Call Shutdown() when
// receiving termination signals to ensure checkpoint data is persisted.
func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
	startupTime := time.Now()

	if rawConfig != nil {
		config.Validate(configSchema, rawConfig)
		dec := json.NewDecoder(bytes.NewReader(rawConfig))
		dec.DisallowUnknownFields()
		if err := dec.Decode(&Keys); err != nil {
			cclog.Abortf("[METRICSTORE]> Metric Store Config Init: Could not decode config file '%s'.\nError: %s\n", rawConfig, err.Error())
		}
	}

	// Set NumWorkers from config or use default
	if Keys.NumWorkers <= 0 {
		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
	}
	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)

	// Pass the config.MetricStoreKeys
	InitMetrics(metrics)

	ms := GetMemoryStore()

	d, err := time.ParseDuration(Keys.RetentionInMemory)
	if err != nil {
		cclog.Fatal(err)
	}

	restoreFrom := startupTime.Add(-d)
	cclog.Infof("[METRICSTORE]> Loading checkpoints newer than %s\n", restoreFrom.Format(time.RFC3339))
	files, err := ms.FromCheckpointFiles(Keys.Checkpoints.RootDir, restoreFrom.Unix())
	loadedData := ms.SizeInBytes() / 1024 / 1024 // In MB
	if err != nil {
		cclog.Fatalf("[METRICSTORE]> Loading checkpoints failed: %s\n", err.Error())
	} else {
		cclog.Infof("[METRICSTORE]> Checkpoints loaded (%d files, %d MB, that took %fs)\n", files, loadedData, time.Since(startupTime).Seconds())
	}

	// Try to use less memory by forcing a GC run here and then
	// lowering the target percentage. The default of 100 means
	// that only once the ratio of new allocations execeds the
	// previously active heap, a GC is triggered.
	// Forcing a GC here will set the "previously active heap"
	// to a minumum.
	// runtime.GC()

	ctx, shutdown := context.WithCancel(context.Background())

	Retention(wg, ctx)
	Checkpointing(wg, ctx)
	CleanUp(wg, ctx)
	WALStaging(wg, ctx)
	MemoryUsageTracker(wg, ctx)

	// Note: Signal handling has been removed from this function.
	// The caller is responsible for handling shutdown signals and calling
	// the shutdown() function when appropriate.
	// Store the shutdown function for later use by Shutdown()
	shutdownFuncMu.Lock()
	shutdownFunc = shutdown
	shutdownFuncMu.Unlock()

	if Keys.Subscriptions != nil {
		wg.Go(func() {
			if err := ReceiveNats(ms, Keys.NumWorkers, ctx); err != nil {
				cclog.Fatal(err)
			}
		})
	}
}

// InitMetrics initializes the singleton MemoryStore instance with the given metric configurations.
//
// This function must be called before GetMemoryStore() and can only be called once due to
// the singleton pattern. It assigns each metric an internal offset for efficient buffer indexing.
//
// Parameters:
//   - metrics: Map of metric names to their configurations (frequency and aggregation strategy)
//
// Panics if any metric has Frequency == 0, which indicates an invalid configuration.
//
// After this call, the global msInstance is ready for use via GetMemoryStore().
func InitMetrics(metrics map[string]MetricConfig) {
	singleton.Do(func() {
		offset := 0
		for key, cfg := range metrics {
			if cfg.Frequency == 0 {
				panic("[METRICSTORE]> invalid frequency")
			}

			metrics[key] = MetricConfig{
				Frequency:   cfg.Frequency,
				Aggregation: cfg.Aggregation,
				offset:      offset,
			}
			offset += 1
		}

		msInstance = &MemoryStore{
			root: Level{
				metrics:  make([]*buffer, len(metrics)),
				children: make(map[string]*Level),
			},
			Metrics: metrics,
		}
	})
}

// GetMemoryStore returns the singleton MemoryStore instance.
//
// Returns the initialized MemoryStore singleton. Calls cclog.Fatal if InitMetrics() was not called first.
//
// This function is safe for concurrent use after initialization.
func GetMemoryStore() *MemoryStore {
	if msInstance == nil {
		cclog.Warnf("[METRICSTORE]> MemoryStore not initialized!")
	}

	return msInstance
}

func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
	if metric, ok := ms.Metrics[metricName]; ok {
		return metric.Frequency, nil
	}
	return 0, fmt.Errorf("[METRICSTORE]> metric %s not found", metricName)
}

// SetNodeProvider sets the NodeProvider implementation for the MemoryStore.
// This must be called during initialization to provide job state information
// for selective buffer retention during Free operations.
// If not set, the Free function will fall back to freeing all buffers.
func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
	ms.nodeProvider = provider
}

// Shutdown performs a graceful shutdown of the metric store.
//
// This function cancels all background goroutines started by Init() and writes
// a final checkpoint to disk before returning. It should be called when the
// application receives a termination signal.
//
// The function will:
//  1. Cancel the context to stop all background workers
//  2. Close the WAL messages channel if using WAL format
//  3. Write a final checkpoint to preserve in-memory data
//  4. Log any errors encountered during shutdown
//
// Note: This function blocks until the final checkpoint is written.
func Shutdown() {
	shutdownFuncMu.Lock()
	defer shutdownFuncMu.Unlock()
	if shutdownFunc != nil {
		shutdownFunc()
	}

	if Keys.Checkpoints.FileFormat == "wal" {
		for _, ch := range walShardChs {
			close(ch)
		}
	}

	cclog.Infof("[METRICSTORE]> Writing to '%s'...\n", Keys.Checkpoints.RootDir)
	var files int
	var err error

	ms := GetMemoryStore()

	lastCheckpointMu.Lock()
	from := lastCheckpoint
	lastCheckpointMu.Unlock()

	if Keys.Checkpoints.FileFormat == "wal" {
		var hostDirs []string
		files, hostDirs, err = ms.ToCheckpointWAL(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
		if err == nil {
			RotateWALFilesAfterShutdown(hostDirs)
		}
	} else {
		files, err = ms.ToCheckpoint(Keys.Checkpoints.RootDir, from.Unix(), time.Now().Unix())
	}

	if err != nil {
		cclog.Errorf("[METRICSTORE]> Writing checkpoint failed: %s\n", err.Error())
	}
	cclog.Infof("[METRICSTORE]> Done! (%d files written)\n", files)
}

// Retention starts a background goroutine that periodically frees old metric data.
//
// This worker runs at half the retention interval and calls Free() to remove buffers
// older than the configured retention time. It respects the NodeProvider to preserve
// data for nodes with active jobs.
//
// Parameters:
//   - wg: WaitGroup to signal completion when context is cancelled
//   - ctx: Context for cancellation signal
//
// The goroutine exits when ctx is cancelled.
func Retention(wg *sync.WaitGroup, ctx context.Context) {
	ms := GetMemoryStore()

	wg.Go(func() {
		d, err := time.ParseDuration(Keys.RetentionInMemory)
		if err != nil {
			cclog.Fatal(err)
		}
		if d <= 0 {
			return
		}

		tickInterval := d / 2
		if tickInterval <= 0 {
			return
		}
		ticker := time.NewTicker(tickInterval)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				state.mu.Lock()

				t := time.Now().Add(-d)

				state.lastRetentionTime = t.Unix()

				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))

				freed, err := Free(ms, t)
				if err != nil {
					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
				} else {
					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
				}

				state.mu.Unlock()

				// Clean up the buffer pool
				bufferPool.Clean(state.lastRetentionTime)
			}
		}
	})
}

// MemoryUsageTracker starts a background goroutine that monitors memory usage.
//
// This worker checks actual process memory usage (via runtime.MemStats) periodically
// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
// performance issues.
//
// The tracker logs both actual memory usage (heap allocated) and metric data size for
// visibility into memory overhead from Go runtime structures and allocations.
//
// Parameters:
//   - wg: WaitGroup to signal completion when context is cancelled
//   - ctx: Context for cancellation signal
//
// The goroutine exits when ctx is cancelled.
func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
	ms := GetMemoryStore()

	wg.Go(func() {
		normalInterval := DefaultMemoryUsageTrackerInterval
		fastInterval := 30 * time.Second

		if normalInterval <= 0 {
			return
		}

		ticker := time.NewTicker(normalInterval)
		defer ticker.Stop()
		currentInterval := normalInterval

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				var mem runtime.MemStats
				runtime.ReadMemStats(&mem)
				actualMemoryGB := float64(mem.Alloc) / 1e9
				metricDataGB := ms.SizeInGB()
				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)

				freedExcluded := 0
				freedEmergency := 0
				var err error

				state.mu.RLock()
				lastRetention := state.lastRetentionTime
				selectorsExcluded := state.selectorsExcluded
				state.mu.RUnlock()

				if lastRetention != 0 && selectorsExcluded {
					freedExcluded, err = ms.Free(nil, lastRetention)
					if err != nil {
						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
					}

					if freedExcluded > 0 {
						debug.FreeOSMemory()
						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
					}
				}

				runtime.ReadMemStats(&mem)
				actualMemoryGB = float64(mem.Alloc) / 1e9

				if actualMemoryGB > float64(Keys.MemoryCap) {
					cclog.Warnf("[METRICSTORE]> memory usage %.2f GB exceeds cap %d GB, starting emergency buffer freeing", actualMemoryGB, Keys.MemoryCap)

					// Use progressive time-based Free with increasing threshold
					// instead of ForceFree loop — fewer tree traversals, more effective
					d, parseErr := time.ParseDuration(Keys.RetentionInMemory)
					if parseErr != nil {
						cclog.Errorf("[METRICSTORE]> cannot parse retention duration: %s", parseErr)
					} else {
						thresholds := []float64{0.75, 0.5, 0.25}
						for _, fraction := range thresholds {
							threshold := time.Now().Add(-time.Duration(float64(d) * fraction))
							freed, freeErr := ms.Free(nil, threshold.Unix())
							if freeErr != nil {
								cclog.Errorf("[METRICSTORE]> error while freeing buffers at %.0f%% retention: %s", fraction*100, freeErr)
							}
							freedEmergency += freed

							bufferPool.Clear()
							runtime.GC()
							runtime.ReadMemStats(&mem)
							actualMemoryGB = float64(mem.Alloc) / 1e9

							if actualMemoryGB < float64(Keys.MemoryCap) {
								break
							}
						}
					}

					bufferPool.Clear()
					debug.FreeOSMemory()

					runtime.ReadMemStats(&mem)
					actualMemoryGB = float64(mem.Alloc) / 1e9

					if actualMemoryGB >= float64(Keys.MemoryCap) {
						cclog.Errorf("[METRICSTORE]> after emergency frees (%d buffers), memory usage %.2f GB still at/above cap %d GB", freedEmergency, actualMemoryGB, Keys.MemoryCap)
					} else {
						cclog.Infof("[METRICSTORE]> emergency freeing complete: %d buffers freed, memory now %.2f GB", freedEmergency, actualMemoryGB)
					}
				}

				// Adaptive ticker: check more frequently when memory is high
				memoryRatio := actualMemoryGB / float64(Keys.MemoryCap)
				if memoryRatio > 0.8 && currentInterval != fastInterval {
					ticker.Reset(fastInterval)
					currentInterval = fastInterval
					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to fast check interval (30s)", memoryRatio*100)
				} else if memoryRatio <= 0.8 && currentInterval != normalInterval {
					ticker.Reset(normalInterval)
					currentInterval = normalInterval
					cclog.Infof("[METRICSTORE]> memory at %.0f%% of cap, switching to normal check interval", memoryRatio*100)
				}
			}
		}
	})
}

// Free removes metric data older than the given time while preserving data for active nodes.
//
// This function implements intelligent retention by consulting the NodeProvider (if configured)
// to determine which nodes are currently in use by running jobs. Data for these nodes is
// preserved even if older than the retention time.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - t: Time threshold - buffers with data older than this will be freed
//
// Returns:
//   - Number of buffers freed
//   - Error if NodeProvider query fails
//
// Behavior:
//   - If no NodeProvider is set: frees all buffers older than t
//   - If NodeProvider returns empty map: frees all buffers older than t
//   - Otherwise: preserves buffers for nodes returned by GetUsedNodes(), frees others
func Free(ms *MemoryStore, t time.Time) (int, error) {
	// If no NodeProvider is configured, free all buffers older than t
	if ms.nodeProvider == nil {
		return ms.Free(nil, t.Unix())
	}

	excludeSelectors, err := ms.nodeProvider.GetUsedNodes(t.Unix())
	if err != nil {
		return 0, err
	}

	switch lenMap := len(excludeSelectors); lenMap {

	// If the length of the map returned by GetUsedNodes() is 0,
	// then use default Free method with nil selector
	case 0:
		state.selectorsExcluded = false
		return ms.Free(nil, t.Unix())

	// Else formulate selectors, exclude those from the map
	// and free the rest of the selectors
	default:
		state.selectorsExcluded = true
		selectors := GetSelectors(ms, excludeSelectors)
		return FreeSelected(ms, selectors, t)
	}
}

// FreeSelected frees buffers for specific selectors while preserving others.
//
// This function is used when we want to retain some specific nodes beyond the retention time.
// It iterates through the provided selectors and frees their associated buffers.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - selectors: List of selector paths to free (e.g., [["cluster1", "node1"], ["cluster2", "node2"]])
//   - t: Time threshold for freeing buffers
//
// Returns the total number of buffers freed and any error encountered.
func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
	freed := 0

	for _, selector := range selectors {

		freedBuffers, err := ms.Free(selector, t.Unix())
		if err != nil {
			cclog.Errorf("error while freeing selected buffers: %#v", err)
		}
		freed += freedBuffers

	}

	return freed, nil
}

// GetSelectors returns all selectors at depth 2 (cluster/node level) that are NOT in the exclusion map.
//
// This function generates a list of selectors whose buffers should be freed by excluding
// selectors that correspond to nodes currently in use by running jobs.
//
// Parameters:
//   - ms: The MemoryStore instance
//   - excludeSelectors: Map of cluster names to node hostnames that should NOT be freed
//
// Returns a list of selectors ([]string paths) that can be safely freed.
//
// Example:
//
//	If the tree has paths ["emmy", "node001"] and ["emmy", "node002"],
//	and excludeSelectors contains {"emmy": ["node001"]},
//	then only [["emmy", "node002"]] is returned.
func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
	allSelectors := ms.GetPaths(2)

	filteredSelectors := make([][]string, 0, len(allSelectors))

	for _, path := range allSelectors {
		if len(path) < 2 {
			continue
		}

		key := path[0]   // The "Key" (Level 1)
		value := path[1] // The "Value" (Level 2)

		exclude := false

		// Check if the key exists in our exclusion map
		if excludedValues, exists := excludeSelectors[key]; exists {
			// The key exists, now check if the specific value is in the exclusion list
			if slices.Contains(excludedValues, value) {
				exclude = true
			}
		}

		if !exclude {
			filteredSelectors = append(filteredSelectors, path)
		}
	}

	return filteredSelectors
}

// GetPaths returns a list of lists (paths) to the specified depth.
func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
	var results [][]string

	// Start recursion. Initial path is empty.
	// We treat Root as depth 0.
	ms.root.collectPaths(0, targetDepth, []string{}, &results)

	return results
}

// Write all values in `metrics` to the level specified by `selector` for time `ts`.
// Look at `findLevelOrCreate` for how selectors work.
func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
	var ok bool
	for i, metric := range metrics {
		if metric.MetricConfig.Frequency == 0 {
			metric.MetricConfig, ok = m.Metrics[metric.Name]
			if !ok {
				cclog.Debugf("[METRICSTORE]> Unknown metric '%s' in Write() - skipping", metric.Name)
				metric.MetricConfig.Frequency = 0
			}
			metrics[i] = metric
		}
	}

	return m.WriteToLevel(&m.root, selector, ts, metrics)
}

func (m *MemoryStore) GetLevel(selector []string) *Level {
	return m.root.findLevelOrCreate(selector, len(m.Metrics))
}

// WriteToLevel assumes that `minfo` in `metrics` is filled in
func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {
	l = l.findLevelOrCreate(selector, len(m.Metrics))
	l.lock.Lock()
	defer l.lock.Unlock()

	for _, metric := range metrics {
		if metric.MetricConfig.Frequency == 0 {
			continue
		}

		b := l.metrics[metric.MetricConfig.offset]
		if b == nil {
			// First write to this metric and level
			b = newBuffer(ts, metric.MetricConfig.Frequency)
			l.metrics[metric.MetricConfig.offset] = b
		}

		nb, err := b.write(ts, metric.Value)
		if err != nil {
			return err
		}

		// Last write created a new buffer...
		if b != nb {
			l.metrics[metric.MetricConfig.offset] = nb
		}
	}
	return nil
}

// Read returns all values for metric `metric` from `from` to `to` for the selected level(s).
// If the level does not hold the metric itself, the data will be aggregated recursively from the children.
// The second and third return value are the actual from/to for the data. Those can be different from
// the range asked for if no data was available.
func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, resolution int64) ([]schema.Float, int64, int64, int64, error) {
	if from > to {
		return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid time range")
	}

	minfo, ok := m.Metrics[metric]
	if !ok {
		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
	}

	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)

	err := m.root.findBuffers(selector, minfo.offset, func(b *buffer) error {
		cdata, cfrom, cto, err := b.read(from, to, data)
		if err != nil {
			return err
		}

		if n == 0 {
			from, to = cfrom, cto
		} else if from != cfrom || to != cto || len(data) != len(cdata) {
			missingfront, missingback := int((from-cfrom)/minfo.Frequency), int((to-cto)/minfo.Frequency)
			if missingfront != 0 {
				return ErrDataDoesNotAlign
			}

			newlen := len(cdata) - missingback
			if newlen < 1 {
				return ErrDataDoesNotAlign
			}
			cdata = cdata[0:newlen]
			if len(cdata) != len(data) {
				return ErrDataDoesNotAlign
			}

			from, to = cfrom, cto
		}

		data = cdata
		n += 1
		return nil
	})

	if err != nil {
		return nil, 0, 0, 0, err
	} else if n == 0 {
		return nil, 0, 0, 0, ErrNoHostOrMetric
	} else if n > 1 {
		if minfo.Aggregation == AvgAggregation {
			normalize := 1. / schema.Float(n)
			for i := 0; i < len(data); i++ {
				data[i] *= normalize
			}
		} else if minfo.Aggregation != SumAggregation {
			return nil, 0, 0, 0, errors.New("[METRICSTORE]> invalid aggregation")
		}
	}

	data, resolution, err = resampler.LargestTriangleThreeBucket(data, minfo.Frequency, resolution)
	if err != nil {
		return nil, 0, 0, 0, err
	}

	return data, from, to, resolution, nil
}

// Free releases all buffers for the selected level and all its children that
// contain only values older than `t`.
func (m *MemoryStore) Free(selector []string, t int64) (int, error) {
	return m.GetLevel(selector).free(t)
}

// ForceFree unconditionally removes the oldest buffer from each metric chain.
func (m *MemoryStore) ForceFree() (int, error) {
	return m.GetLevel(nil).forceFree()
}

func (m *MemoryStore) FreeAll() error {
	m.root.lock.Lock()
	defer m.root.lock.Unlock()

	for k := range m.root.children {
		delete(m.root.children, k)
	}

	return nil
}

func (m *MemoryStore) SizeInBytes() int64 {
	return m.root.sizeInBytes()
}

func (m *MemoryStore) SizeInGB() float64 {
	return float64(m.root.sizeInBytes()) / 1e9
}

// ListChildren , given a selector, returns a list of all children of the level
// selected.
func (m *MemoryStore) ListChildren(selector []string) []string {
	lvl := &m.root
	for lvl != nil && len(selector) != 0 {
		lvl.lock.RLock()
		next := lvl.children[selector[0]]
		lvl.lock.RUnlock()
		lvl = next
		selector = selector[1:]
	}

	if lvl == nil {
		return nil
	}

	lvl.lock.RLock()
	defer lvl.lock.RUnlock()

	children := make([]string, 0, len(lvl.children))
	for child := range lvl.children {
		children = append(children, child)
	}

	return children
}