     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements high-level query functions for loading job metric data
     7→// with automatic scope transformation and aggregation.
     8→//
     9→// Key Concepts:
    10→//
    11→// Metric Scopes: Metrics are collected at different granularities (native scope):
    12→//   - HWThread: Per hardware thread
    13→//   - Core: Per CPU core
    14→//   - Socket: Per CPU socket
    15→//   - MemoryDomain: Per memory domain (NUMA)
    16→//   - Accelerator: Per GPU/accelerator
    17→//   - Node: Per compute node
    18→//
    19→// Scope Transformation: The buildQueries functions transform between native scope
    20→// and requested scope by:
    21→//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
    22→//   - Rejecting requests for finer granularity than available
    23→//   - Handling special cases (e.g., Accelerator metrics)
    24→//
    25→// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
    26→// based on cluster topology and job resources.
    27→package metricstore
    28→
    29→import (
    30→	"context"
    31→	"fmt"
    32→	"strings"
    33→	"time"
    34→
    35→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    36→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    37→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    38→)
    39→
    40→type InternalMetricStore struct{}
    41→
    42→var MetricStoreHandle *InternalMetricStore
    43→
    44→// HealthCheck delegates to the internal MemoryStore's HealthCheck.
    45→func (ccms *InternalMetricStore) HealthCheck(cluster string,
    46→	nodes []string, metrics []string,
    47→) (map[string]HealthCheckResult, error) {
    48→	return GetMemoryStore().HealthCheck(cluster, nodes, metrics)
    49→}
    50→
    51→// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
    52→// When set to a non-nil function, LoadData will call this function instead of the default implementation.
    53→var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)
    54→
    55→// LoadData loads metric data for a specific job with automatic scope transformation.
    56→//
    57→// This is the primary function for retrieving job metric data. It handles:
    58→//   - Building queries with scope transformation via buildQueries
    59→//   - Fetching data from the metric store
    60→//   - Organizing results by metric and scope
    61→//   - Converting NaN statistics to 0 for JSON compatibility
    62→//   - Partial error handling (returns data for successful queries even if some fail)
    63→//
    64→// Parameters:
    65→//   - job: Job metadata including cluster, resources, and time range
    66→//   - metrics: List of metric names to load
    67→//   - scopes: Requested metric scopes (will be transformed to match native scopes)
    68→//   - ctx: Context for cancellation (currently unused but reserved for future use)
    69→//   - resolution: Data resolution in seconds (0 for native resolution)
    70→//
    71→// Returns:
    72→//   - JobData: Map of metric → scope → JobMetric with time-series data and statistics
    73→//   - Error: Returns error if query building or fetching fails, or partial error listing failed hosts
    74→//
    75→// Example:
    76→//
    77→//	jobData, err := LoadData(job, []string{"cpu_load", "mem_used"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 60)
    78→func (ccms *InternalMetricStore) LoadData(
    79→	job *schema.Job,
    80→	metrics []string,
    81→	scopes []schema.MetricScope,
    82→	ctx context.Context,
    83→	resolution int,
    84→) (schema.JobData, error) {
    85→	if TestLoadDataCallback != nil {
    86→		return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
    87→	}
    88→
    89→	queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
    90→	if err != nil {
    91→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
    92→		return nil, err
    93→	}
    94→
    95→	// Verify assignment is correct - log any inconsistencies for debugging
    96→	if len(queries) != len(assignedScope) {
    97→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
    98→			len(queries), len(assignedScope))
    99→	}
   100→
   101→	req := APIQueryRequest{
   102→		Cluster:   job.Cluster,
   103→		From:      job.StartTime,
   104→		To:        job.StartTime + int64(job.Duration),
   105→		Queries:   queries,
   106→		WithStats: true,
   107→		WithData:  true,
   108→	}
   109→
   110→	resBody, err := FetchData(req)
   111→	if err != nil {
   112→		cclog.Errorf("Error while fetching data : %s", err.Error())
   113→		return nil, err
   114→	}
   115→
   116→	var errors []string
   117→	jobData := make(schema.JobData)
   118→
   119→	// Add safety check for potential index out of range errors
   120→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   121→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   122→			len(req.Queries), len(resBody.Results), len(assignedScope))
   123→		if len(resBody.Results) > len(req.Queries) {
   124→			resBody.Results = resBody.Results[:len(req.Queries)]
   125→		}
   126→		if len(assignedScope) > len(req.Queries) {
   127→			assignedScope = assignedScope[:len(req.Queries)]
   128→		}
   129→	}
   130→
   131→	for i, row := range resBody.Results {
   132→		query := req.Queries[i]
   133→		metric := query.Metric
   134→		scope := assignedScope[i]
   135→		mc := archive.GetMetricConfig(job.Cluster, metric)
   136→
   137→		if mc == nil {
   138→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   139→			continue
   140→		}
   141→
   142→		if _, ok := jobData[metric]; !ok {
   143→			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
   144→		}
   145→
   146→		res := mc.Timestep
   147→		if len(row) > 0 {
   148→			res = int(row[0].Resolution)
   149→		}
   150→
   151→		jobMetric, ok := jobData[metric][scope]
   152→		if !ok {
   153→			jobMetric = &schema.JobMetric{
   154→				Unit:     mc.Unit,
   155→				Timestep: res,
   156→				Series:   make([]schema.Series, 0),
   157→			}
   158→			jobData[metric][scope] = jobMetric
   159→		}
   160→
   161→		for ndx, res := range row {
   162→			if res.Error != nil {
   163→				/* Build list for "partial errors", if any */
   164→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   165→				continue
   166→			}
   167→
   168→			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)
   169→
   170→			SanitizeStats(&res.Avg, &res.Min, &res.Max)
   171→
   172→			jobMetric.Series = append(jobMetric.Series, schema.Series{
   173→				Hostname: query.Hostname,
   174→				ID:       id,
   175→				Statistics: schema.MetricStatistics{
   176→					Avg: float64(res.Avg),
   177→					Min: float64(res.Min),
   178→					Max: float64(res.Max),
   179→				},
   180→				Data: res.Data,
   181→			})
   182→		}
   183→
   184→		// So that one can later check len(jobData):
   185→		if len(jobMetric.Series) == 0 {
   186→			delete(jobData[metric], scope)
   187→			if len(jobData[metric]) == 0 {
   188→				delete(jobData, metric)
   189→			}
   190→		}
   191→	}
   192→
   193→	if len(errors) != 0 {
   194→		/* Returns list for "partial errors" */
   195→		return jobData, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   196→	}
   197→	return jobData, nil
   198→}
   199→
   200→// buildQueries constructs APIQuery structures with automatic scope transformation for a job.
   201→//
   202→// This function implements the core scope transformation logic, handling all combinations of
   203→// native metric scopes and requested scopes. It uses the cluster topology to determine which
   204→// hardware IDs to include in each query.
   205→//
   206→// Scope Transformation Rules:
   207→//   - If native scope >= requested scope: Aggregates data (Aggregate=true in APIQuery)
   208→//   - If native scope < requested scope: Returns error (cannot increase granularity)
   209→//   - Special handling for Accelerator scope (independent of CPU hierarchy)
   210→//
   211→// The function generates one or more APIQuery per (metric, scope, host) combination:
   212→//   - For non-aggregated queries: One query with all relevant IDs
   213→//   - For aggregated queries: May generate multiple queries (e.g., one per socket/core)
   214→//
   215→// Parameters:
   216→//   - job: Job metadata including cluster, subcluster, and resource allocation
   217→//   - metrics: List of metrics to query
   218→//   - scopes: Requested scopes for each metric
   219→//   - resolution: Data resolution in seconds
   220→//
   221→// Returns:
   222→//   - []APIQuery: List of queries to execute
   223→//   - []schema.MetricScope: Assigned scope for each query (after transformation)
   224→//   - error: Returns error if topology lookup fails or unhandled scope combination encountered
   225→func buildQueries(
   226→	job *schema.Job,
   227→	metrics []string,
   228→	scopes []schema.MetricScope,
   229→	resolution int64,
   230→) ([]APIQuery, []schema.MetricScope, error) {
   231→	if len(job.Resources) == 0 {
   232→		return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > no resources allocated for job %d", job.JobID)
   233→	}
   234→
   235→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
   236→	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(job.Resources))
   237→
   238→	subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster)
   239→	if scerr != nil {
   240→		return nil, nil, scerr
   241→	}
   242→	topology := subcluster.Topology
   243→
   244→	for _, metric := range metrics {
   245→		mc := archive.GetMetricConfig(job.Cluster, metric)
   246→		if mc == nil {
   247→			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
   248→			continue
   249→		}
   250→
   251→		// Skip if metric is removed for subcluster
   252→		if len(mc.SubClusters) != 0 && IsMetricRemovedForSubCluster(mc, job.SubCluster) {
   253→			continue
   254→		}
   255→
   256→		// Avoid duplicates...
   257→		handledScopes := make([]schema.MetricScope, 0, 3)
   258→
   259→	scopesLoop:
   260→		for _, requestedScope := range scopes {
   261→			nativeScope := mc.Scope
   262→			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
   263→				continue
   264→			}
   265→
   266→			scope := nativeScope.Max(requestedScope)
   267→			for _, s := range handledScopes {
   268→				if scope == s {
   269→					continue scopesLoop
   270→				}
   271→			}
   272→			handledScopes = append(handledScopes, scope)
   273→
   274→			for _, host := range job.Resources {
   275→				hwthreads := host.HWThreads
   276→				if hwthreads == nil {
   277→					hwthreads = topology.Node
   278→				}
   279→
   280→				scopeResults, ok := BuildScopeQueries(
   281→					nativeScope, requestedScope,
   282→					metric, host.Hostname,
   283→					&topology, hwthreads, host.Accelerators,
   284→				)
   285→
   286→				if !ok {
   287→					return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   288→				}
   289→
   290→				for _, sr := range scopeResults {
   291→					queries = append(queries, APIQuery{
   292→						Metric:     sr.Metric,
   293→						Hostname:   sr.Hostname,
   294→						Aggregate:  sr.Aggregate,
   295→						Type:       sr.Type,
   296→						TypeIds:    sr.TypeIds,
   297→						Resolution: resolution,
   298→					})
   299→					assignedScope = append(assignedScope, sr.Scope)
   300→				}
   301→			}
   302→		}
   303→	}
   304→
   305→	return queries, assignedScope, nil
   306→}
   307→
   308→// LoadStats loads only metric statistics (avg/min/max) for a job at node scope.
   309→//
   310→// This is an optimized version of LoadData that fetches only statistics without
   311→// time-series data, reducing bandwidth and memory usage. Always queries at node scope.
   312→//
   313→// Parameters:
   314→//   - job: Job metadata
   315→//   - metrics: List of metric names
   316→//   - ctx: Context (currently unused)
   317→//
   318→// Returns:
   319→//   - Map of metric → hostname → statistics
   320→//   - Error on query building or fetching failure
   321→func (ccms *InternalMetricStore) LoadStats(
   322→	job *schema.Job,
   323→	metrics []string,
   324→	ctx context.Context,
   325→) (map[string]map[string]schema.MetricStatistics, error) {
   326→	// TODO(#166): Add scope parameter for analysis view accelerator normalization
   327→	queries, _, err := buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0)
   328→	if err != nil {
   329→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
   330→		return nil, err
   331→	}
   332→
   333→	req := APIQueryRequest{
   334→		Cluster:   job.Cluster,
   335→		From:      job.StartTime,
   336→		To:        job.StartTime + int64(job.Duration),
   337→		Queries:   queries,
   338→		WithStats: true,
   339→		WithData:  false,
   340→	}
   341→
   342→	resBody, err := FetchData(req)
   343→	if err != nil {
   344→		cclog.Errorf("Error while fetching data : %s", err.Error())
   345→		return nil, err
   346→	}
   347→
   348→	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
   349→	for i, res := range resBody.Results {
   350→		if i >= len(req.Queries) {
   351→			cclog.Warnf("LoadStats: result index %d exceeds queries length %d", i, len(req.Queries))
   352→			break
   353→		}
   354→		if len(res) == 0 {
   355→			// No Data Found For Metric, Logged in FetchData to Warn
   356→			continue
   357→		}
   358→		query := req.Queries[i]
   359→		metric := query.Metric
   360→		data := res[0]
   361→		if data.Error != nil {
   362→			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
   363→			continue
   364→		}
   365→
   366→		metricdata, ok := stats[metric]
   367→		if !ok {
   368→			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
   369→			stats[metric] = metricdata
   370→		}
   371→
   372→		if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() {
   373→			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
   374→			continue
   375→		}
   376→
   377→		metricdata[query.Hostname] = schema.MetricStatistics{
   378→			Avg: float64(data.Avg),
   379→			Min: float64(data.Min),
   380→			Max: float64(data.Max),
   381→		}
   382→	}
   383→
   384→	return stats, nil
   385→}
   386→
   387→// LoadScopedStats loads metric statistics for a job with scope-aware grouping.
   388→//
   389→// Similar to LoadStats but supports multiple scopes and returns statistics grouped
   390→// by scope with hardware IDs (e.g., per-core, per-socket statistics).
   391→//
   392→// Parameters:
   393→//   - job: Job metadata
   394→//   - metrics: List of metric names
   395→//   - scopes: Requested metric scopes
   396→//   - ctx: Context (currently unused)
   397→//
   398→// Returns:
   399→//   - ScopedJobStats: Map of metric → scope → []ScopedStats (with hostname and ID)
   400→//   - Error or partial error listing failed queries
   401→func (ccms *InternalMetricStore) LoadScopedStats(
   402→	job *schema.Job,
   403→	metrics []string,
   404→	scopes []schema.MetricScope,
   405→	ctx context.Context,
   406→) (schema.ScopedJobStats, error) {
   407→	queries, assignedScope, err := buildQueries(job, metrics, scopes, 0)
   408→	if err != nil {
   409→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   410→		return nil, err
   411→	}
   412→
   413→	req := APIQueryRequest{
   414→		Cluster:   job.Cluster,
   415→		From:      job.StartTime,
   416→		To:        job.StartTime + int64(job.Duration),
   417→		Queries:   queries,
   418→		WithStats: true,
   419→		WithData:  false,
   420→	}
   421→
   422→	resBody, err := FetchData(req)
   423→	if err != nil {
   424→		cclog.Errorf("Error while fetching data : %s", err.Error())
   425→		return nil, err
   426→	}
   427→
   428→	var errors []string
   429→	scopedJobStats := make(schema.ScopedJobStats)
   430→
   431→	for i, row := range resBody.Results {
   432→		if len(row) == 0 {
   433→			// No Data Found For Metric, Logged in FetchData to Warn
   434→			continue
   435→		}
   436→		query := req.Queries[i]
   437→		metric := query.Metric
   438→		scope := assignedScope[i]
   439→
   440→		if _, ok := scopedJobStats[metric]; !ok {
   441→			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
   442→		}
   443→
   444→		if _, ok := scopedJobStats[metric][scope]; !ok {
   445→			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
   446→		}
   447→
   448→		for ndx, res := range row {
   449→			if res.Error != nil {
   450→				/* Build list for "partial errors", if any */
   451→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   452→				continue
   453→			}
   454→
   455→			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)
   456→
   457→			SanitizeStats(&res.Avg, &res.Min, &res.Max)
   458→
   459→			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
   460→				Hostname: query.Hostname,
   461→				ID:       id,
   462→				Data: &schema.MetricStatistics{
   463→					Avg: float64(res.Avg),
   464→					Min: float64(res.Min),
   465→					Max: float64(res.Max),
   466→				},
   467→			})
   468→		}
   469→
   470→		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
   471→		if len(scopedJobStats[metric][scope]) == 0 {
   472→			delete(scopedJobStats[metric], scope)
   473→			if len(scopedJobStats[metric]) == 0 {
   474→				delete(scopedJobStats, metric)
   475→			}
   476→		}
   477→	}
   478→
   479→	if len(errors) != 0 {
   480→		/* Returns list for "partial errors" */
   481→		return scopedJobStats, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   482→	}
   483→	return scopedJobStats, nil
   484→}
   485→
   486→// LoadNodeData loads metric data for specific nodes in a cluster over a time range.
   487→//
   488→// Unlike LoadData which operates on job resources, this function queries arbitrary nodes
   489→// directly. Useful for system monitoring and node status views.
   490→//
   491→// Parameters:
   492→//   - cluster: Cluster name
   493→//   - metrics: List of metric names
   494→//   - nodes: List of node hostnames (nil = all nodes in cluster via ForAllNodes)
   495→//   - scopes: Requested metric scopes (currently unused - always node scope)
   496→//   - from, to: Time range
   497→//   - ctx: Context (currently unused)
   498→//
   499→// Returns:
   500→//   - Map of hostname → metric → []JobMetric
   501→//   - Error or partial error listing failed queries
   502→func (ccms *InternalMetricStore) LoadNodeData(
   503→	cluster string,
   504→	metrics, nodes []string,
   505→	scopes []schema.MetricScope,
   506→	from, to time.Time,
   507→	ctx context.Context,
   508→) (map[string]map[string][]*schema.JobMetric, error) {
   509→	req := APIQueryRequest{
   510→		Cluster:   cluster,
   511→		From:      from.Unix(),
   512→		To:        to.Unix(),
   513→		WithStats: true,
   514→		WithData:  true,
   515→	}
   516→
   517→	if nodes == nil {
   518→		req.ForAllNodes = append(req.ForAllNodes, metrics...)
   519→	} else {
   520→		for _, node := range nodes {
   521→			for _, metric := range metrics {
   522→				req.Queries = append(req.Queries, APIQuery{
   523→					Hostname:   node,
   524→					Metric:     metric,
   525→					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
   526→				})
   527→			}
   528→		}
   529→	}
   530→
   531→	resBody, err := FetchData(req)
   532→	if err != nil {
   533→		cclog.Errorf("Error while fetching data : %s", err.Error())
   534→		return nil, err
   535→	}
   536→
   537→	var errors []string
   538→	data := make(map[string]map[string][]*schema.JobMetric)
   539→	for i, res := range resBody.Results {
   540→		if len(res) == 0 {
   541→			// No Data Found For Metric, Logged in FetchData to Warn
   542→			continue
   543→		}
   544→
   545→		var query APIQuery
   546→		if resBody.Queries != nil {
   547→			query = resBody.Queries[i]
   548→		} else {
   549→			query = req.Queries[i]
   550→		}
   551→
   552→		metric := query.Metric
   553→		qdata := res[0]
   554→		if qdata.Error != nil {
   555→			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
   556→			continue
   557→		}
   558→
   559→		mc := archive.GetMetricConfig(cluster, metric)
   560→		if mc == nil {
   561→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   562→			continue
   563→		}
   564→
   565→		SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)
   566→
   567→		hostdata, ok := data[query.Hostname]
   568→		if !ok {
   569→			hostdata = make(map[string][]*schema.JobMetric)
   570→			data[query.Hostname] = hostdata
   571→		}
   572→
   573→		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
   574→			Unit:     mc.Unit,
   575→			Timestep: mc.Timestep,
   576→			Series: []schema.Series{
   577→				{
   578→					Hostname: query.Hostname,
   579→					Data:     qdata.Data,
   580→					Statistics: schema.MetricStatistics{
   581→						Avg: float64(qdata.Avg),
   582→						Min: float64(qdata.Min),
   583→						Max: float64(qdata.Max),
   584→					},
   585→				},
   586→			},
   587→		})
   588→	}
   589→
   590→	if len(errors) != 0 {
   591→		/* Returns list of "partial errors" */
   592→		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   593→	}
   594→
   595→	return data, nil
   596→}
   597→
   598→// LoadNodeListData loads metric data for a list of nodes with full scope transformation support.
   599→//
   600→// This is the most flexible node data loading function, supporting arbitrary scopes and
   601→// resolution. Uses buildNodeQueries for proper scope transformation based on topology.
   602→//
   603→// Parameters:
   604→//   - cluster: Cluster name
   605→//   - subCluster: SubCluster name (empty string to infer from node names)
   606→//   - nodes: List of node hostnames
   607→//   - metrics: List of metric names
   608→//   - scopes: Requested metric scopes
   609→//   - resolution: Data resolution in seconds
   610→//   - from, to: Time range
   611→//   - ctx: Context (currently unused)
   612→//
   613→// Returns:
   614→//   - Map of hostname → JobData (metric → scope → JobMetric)
   615→//   - Error or partial error listing failed queries
   616→func (ccms *InternalMetricStore) LoadNodeListData(
   617→	cluster, subCluster string,
   618→	nodes []string,
   619→	metrics []string,
   620→	scopes []schema.MetricScope,
   621→	resolution int,
   622→	from, to time.Time,
   623→	ctx context.Context,
   624→) (map[string]schema.JobData, error) {
   625→	// Note: Order of node data is not guaranteed after this point
   626→	queries, assignedScope, err := buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution))
   627→	if err != nil {
   628→		cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
   629→		return nil, err
   630→	}
   631→
   632→	// Verify assignment is correct - log any inconsistencies for debugging
   633→	if len(queries) != len(assignedScope) {
   634→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
   635→			len(queries), len(assignedScope))
   636→	}
   637→
   638→	req := APIQueryRequest{
   639→		Cluster:   cluster,
   640→		Queries:   queries,
   641→		From:      from.Unix(),
   642→		To:        to.Unix(),
   643→		WithStats: true,
   644→		WithData:  true,
   645→	}
   646→
   647→	resBody, err := FetchData(req)
   648→	if err != nil {
   649→		cclog.Errorf("Error while fetching data : %s", err.Error())
   650→		return nil, err
   651→	}
   652→
   653→	var errors []string
   654→	data := make(map[string]schema.JobData)
   655→
   656→	// Add safety check for index out of range issues
   657→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   658→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   659→			len(req.Queries), len(resBody.Results), len(assignedScope))
   660→		if len(resBody.Results) > len(req.Queries) {
   661→			resBody.Results = resBody.Results[:len(req.Queries)]
   662→		}
   663→		if len(assignedScope) > len(req.Queries) {
   664→			assignedScope = assignedScope[:len(req.Queries)]
   665→		}
   666→	}
   667→
   668→	for i, row := range resBody.Results {
   669→		var query APIQuery
   670→		if resBody.Queries != nil {
   671→			if i < len(resBody.Queries) {
   672→				query = resBody.Queries[i]
   673→			} else {
   674→				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
   675→					i, len(resBody.Queries))
   676→				continue
   677→			}
   678→		} else {
   679→			query = req.Queries[i]
   680→		}
   681→
   682→		metric := query.Metric
   683→		scope := assignedScope[i]
   684→		mc := archive.GetMetricConfig(cluster, metric)
   685→		if mc == nil {
   686→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   687→			continue
   688→		}
   689→
   690→		res := mc.Timestep
   691→		if len(row) > 0 {
   692→			res = int(row[0].Resolution)
   693→		}
   694→
   695→		// Init Nested Map Data Structures If Not Found
   696→		hostData, ok := data[query.Hostname]
   697→		if !ok {
   698→			hostData = make(schema.JobData)
   699→			data[query.Hostname] = hostData
   700→		}
   701→
   702→		metricData, ok := hostData[metric]
   703→		if !ok {
   704→			metricData = make(map[schema.MetricScope]*schema.JobMetric)
   705→			data[query.Hostname][metric] = metricData
   706→		}
   707→
   708→		scopeData, ok := metricData[scope]
   709→		if !ok {
   710→			scopeData = &schema.JobMetric{
   711→				Unit:     mc.Unit,
   712→				Timestep: res,
   713→				Series:   make([]schema.Series, 0),
   714→			}
   715→			data[query.Hostname][metric][scope] = scopeData
   716→		}
   717→
   718→		for ndx, res := range row {
   719→			if res.Error != nil {
   720→				/* Build list for "partial errors", if any */
   721→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   722→				continue
   723→			}
   724→
   725→			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)
   726→
   727→			SanitizeStats(&res.Avg, &res.Min, &res.Max)
   728→
   729→			scopeData.Series = append(scopeData.Series, schema.Series{
   730→				Hostname: query.Hostname,
   731→				ID:       id,
   732→				Statistics: schema.MetricStatistics{
   733→					Avg: float64(res.Avg),
   734→					Min: float64(res.Min),
   735→					Max: float64(res.Max),
   736→				},
   737→				Data: res.Data,
   738→			})
   739→		}
   740→	}
   741→
   742→	if len(errors) != 0 {
   743→		/* Returns list of "partial errors" */
   744→		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   745→	}
   746→
   747→	return data, nil
   748→}
   749→
   750→// buildNodeQueries constructs APIQuery structures for node-based queries with scope transformation.
   751→//
   752→// Similar to buildQueries but operates on node lists rather than job resources.
   753→// Supports dynamic subcluster lookup when subCluster parameter is empty.
   754→//
   755→// Parameters:
   756→//   - cluster: Cluster name
   757→//   - subCluster: SubCluster name (empty = infer from node hostnames)
   758→//   - nodes: List of node hostnames
   759→//   - metrics: List of metric names
   760→//   - scopes: Requested metric scopes
   761→//   - resolution: Data resolution in seconds
   762→//
   763→// Returns:
   764→//   - []APIQuery: List of queries to execute
   765→//   - []schema.MetricScope: Assigned scope for each query
   766→//   - error: Returns error if topology lookup fails or unhandled scope combination
   767→func buildNodeQueries(
   768→	cluster string,
   769→	subCluster string,
   770→	nodes []string,
   771→	metrics []string,
   772→	scopes []schema.MetricScope,
   773→	resolution int64,
   774→) ([]APIQuery, []schema.MetricScope, error) {
   775→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
   776→	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(nodes))
   777→
   778→	// Get Topol before loop if subCluster given
   779→	var subClusterTopol *schema.SubCluster
   780→	var scterr error
   781→	if subCluster != "" {
   782→		subClusterTopol, scterr = archive.GetSubCluster(cluster, subCluster)
   783→		if scterr != nil {
   784→			cclog.Errorf("could not load cluster %s subCluster %s topology: %s", cluster, subCluster, scterr.Error())
   785→			return nil, nil, scterr
   786→		}
   787→	}
   788→
   789→	for _, metric := range metrics {
   790→		mc := archive.GetMetricConfig(cluster, metric)
   791→		if mc == nil {
   792→			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
   793→			continue
   794→		}
   795→
   796→		// Skip if metric is removed for subcluster
   797→		if mc.SubClusters != nil && IsMetricRemovedForSubCluster(mc, subCluster) {
   798→			continue
   799→		}
   800→
   801→		// Avoid duplicates...
   802→		handledScopes := make([]schema.MetricScope, 0, 3)
   803→
   804→	nodeScopesLoop:
   805→		for _, requestedScope := range scopes {
   806→			nativeScope := mc.Scope
   807→
   808→			scope := nativeScope.Max(requestedScope)
   809→			for _, s := range handledScopes {
   810→				if scope == s {
   811→					continue nodeScopesLoop
   812→				}
   813→			}
   814→			handledScopes = append(handledScopes, scope)
   815→
   816→			for _, hostname := range nodes {
   817→
   818→				// If no subCluster given, get it by node
   819→				if subCluster == "" {
   820→					subClusterName, scnerr := archive.GetSubClusterByNode(cluster, hostname)
   821→					if scnerr != nil {
   822→						return nil, nil, scnerr
   823→					}
   824→					subClusterTopol, scterr = archive.GetSubCluster(cluster, subClusterName)
   825→					if scterr != nil {
   826→						return nil, nil, scterr
   827→					}
   828→				}
   829→
   830→				// Always full node hwthread id list, no partial queries expected
   831→				topology := subClusterTopol.Topology
   832→				acceleratorIds := topology.GetAcceleratorIDs()
   833→
   834→				// Moved check here if metric matches hardware specs
   835→				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
   836→					continue
   837→				}
   838→
   839→				scopeResults, ok := BuildScopeQueries(
   840→					nativeScope, requestedScope,
   841→					metric, hostname,
   842→					&topology, topology.Node, acceleratorIds,
   843→				)
   844→
   845→				if !ok {
   846→					return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   847→				}
   848→
   849→				for _, sr := range scopeResults {
   850→					queries = append(queries, APIQuery{
   851→						Metric:     sr.Metric,
   852→						Hostname:   sr.Hostname,
   853→						Aggregate:  sr.Aggregate,
   854→						Type:       sr.Type,
   855→						TypeIds:    sr.TypeIds,
   856→						Resolution: resolution,
   857→					})
   858→					assignedScope = append(assignedScope, sr.Scope)
   859→				}
   860→			}
   861→		}
   862→	}
   863→
   864→	return queries, assignedScope, nil
   865→}
   866→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements high-level query functions for loading job metric data
// with automatic scope transformation and aggregation.
//
// Key Concepts:
//
// Metric Scopes: Metrics are collected at different granularities (native scope):
//   - HWThread: Per hardware thread
//   - Core: Per CPU core
//   - Socket: Per CPU socket
//   - MemoryDomain: Per memory domain (NUMA)
//   - Accelerator: Per GPU/accelerator
//   - Node: Per compute node
//
// Scope Transformation: The buildQueries functions transform between native scope
// and requested scope by:
//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
//   - Rejecting requests for finer granularity than available
//   - Handling special cases (e.g., Accelerator metrics)
//
// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
// based on cluster topology and job resources.
package metricstore

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

type InternalMetricStore struct{}

var MetricStoreHandle *InternalMetricStore

// HealthCheck delegates to the internal MemoryStore's HealthCheck.
func (ccms *InternalMetricStore) HealthCheck(cluster string,
	nodes []string, metrics []string,
) (map[string]HealthCheckResult, error) {
	return GetMemoryStore().HealthCheck(cluster, nodes, metrics)
}

// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
// When set to a non-nil function, LoadData will call this function instead of the default implementation.
var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)

// LoadData loads metric data for a specific job with automatic scope transformation.
//
// This is the primary function for retrieving job metric data. It handles:
//   - Building queries with scope transformation via buildQueries
//   - Fetching data from the metric store
//   - Organizing results by metric and scope
//   - Converting NaN statistics to 0 for JSON compatibility
//   - Partial error handling (returns data for successful queries even if some fail)
//
// Parameters:
//   - job: Job metadata including cluster, resources, and time range
//   - metrics: List of metric names to load
//   - scopes: Requested metric scopes (will be transformed to match native scopes)
//   - ctx: Context for cancellation (currently unused but reserved for future use)
//   - resolution: Data resolution in seconds (0 for native resolution)
//
// Returns:
//   - JobData: Map of metric → scope → JobMetric with time-series data and statistics
//   - Error: Returns error if query building or fetching fails, or partial error listing failed hosts
//
// Example:
//
//	jobData, err := LoadData(job, []string{"cpu_load", "mem_used"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 60)
func (ccms *InternalMetricStore) LoadData(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
	resolution int,
) (schema.JobData, error) {
	if TestLoadDataCallback != nil {
		return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
	}

	queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  true,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	jobData := make(schema.JobData)

	// Add safety check for potential index out of range errors
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(job.Cluster, metric)

		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
			continue
		}

		if _, ok := jobData[metric]; !ok {
			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = int(row[0].Resolution)
		}

		jobMetric, ok := jobData[metric][scope]
		if !ok {
			jobMetric = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			jobData[metric][scope] = jobMetric
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)

			SanitizeStats(&res.Avg, &res.Min, &res.Max)

			jobMetric.Series = append(jobMetric.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}

		// So that one can later check len(jobData):
		if len(jobMetric.Series) == 0 {
			delete(jobData[metric], scope)
			if len(jobData[metric]) == 0 {
				delete(jobData, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return jobData, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return jobData, nil
}

// buildQueries constructs APIQuery structures with automatic scope transformation for a job.
//
// This function implements the core scope transformation logic, handling all combinations of
// native metric scopes and requested scopes. It uses the cluster topology to determine which
// hardware IDs to include in each query.
//
// Scope Transformation Rules:
//   - If native scope >= requested scope: Aggregates data (Aggregate=true in APIQuery)
//   - If native scope < requested scope: Returns error (cannot increase granularity)
//   - Special handling for Accelerator scope (independent of CPU hierarchy)
//
// The function generates one or more APIQuery per (metric, scope, host) combination:
//   - For non-aggregated queries: One query with all relevant IDs
//   - For aggregated queries: May generate multiple queries (e.g., one per socket/core)
//
// Parameters:
//   - job: Job metadata including cluster, subcluster, and resource allocation
//   - metrics: List of metrics to query
//   - scopes: Requested scopes for each metric
//   - resolution: Data resolution in seconds
//
// Returns:
//   - []APIQuery: List of queries to execute
//   - []schema.MetricScope: Assigned scope for each query (after transformation)
//   - error: Returns error if topology lookup fails or unhandled scope combination encountered
func buildQueries(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int64,
) ([]APIQuery, []schema.MetricScope, error) {
	if len(job.Resources) == 0 {
		return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > no resources allocated for job %d", job.JobID)
	}

	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(job.Resources))

	subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster)
	if scerr != nil {
		return nil, nil, scerr
	}
	topology := subcluster.Topology

	for _, metric := range metrics {
		mc := archive.GetMetricConfig(job.Cluster, metric)
		if mc == nil {
			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
			continue
		}

		// Skip if metric is removed for subcluster
		if len(mc.SubClusters) != 0 && IsMetricRemovedForSubCluster(mc, job.SubCluster) {
			continue
		}

		// Avoid duplicates...
		handledScopes := make([]schema.MetricScope, 0, 3)

	scopesLoop:
		for _, requestedScope := range scopes {
			nativeScope := mc.Scope
			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
				continue
			}

			scope := nativeScope.Max(requestedScope)
			for _, s := range handledScopes {
				if scope == s {
					continue scopesLoop
				}
			}
			handledScopes = append(handledScopes, scope)

			for _, host := range job.Resources {
				hwthreads := host.HWThreads
				if hwthreads == nil {
					hwthreads = topology.Node
				}

				scopeResults, ok := BuildScopeQueries(
					nativeScope, requestedScope,
					metric, host.Hostname,
					&topology, hwthreads, host.Accelerators,
				)

				if !ok {
					return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
				}

				for _, sr := range scopeResults {
					queries = append(queries, APIQuery{
						Metric:     sr.Metric,
						Hostname:   sr.Hostname,
						Aggregate:  sr.Aggregate,
						Type:       sr.Type,
						TypeIds:    sr.TypeIds,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, sr.Scope)
				}
			}
		}
	}

	return queries, assignedScope, nil
}

// LoadStats loads only metric statistics (avg/min/max) for a job at node scope.
//
// This is an optimized version of LoadData that fetches only statistics without
// time-series data, reducing bandwidth and memory usage. Always queries at node scope.
//
// Parameters:
//   - job: Job metadata
//   - metrics: List of metric names
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of metric → hostname → statistics
//   - Error on query building or fetching failure
func (ccms *InternalMetricStore) LoadStats(
	job *schema.Job,
	metrics []string,
	ctx context.Context,
) (map[string]map[string]schema.MetricStatistics, error) {
	// TODO(#166): Add scope parameter for analysis view accelerator normalization
	queries, _, err := buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
	for i, res := range resBody.Results {
		if i >= len(req.Queries) {
			cclog.Warnf("LoadStats: result index %d exceeds queries length %d", i, len(req.Queries))
			break
		}
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}
		query := req.Queries[i]
		metric := query.Metric
		data := res[0]
		if data.Error != nil {
			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
			continue
		}

		metricdata, ok := stats[metric]
		if !ok {
			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
			stats[metric] = metricdata
		}

		if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() {
			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
			continue
		}

		metricdata[query.Hostname] = schema.MetricStatistics{
			Avg: float64(data.Avg),
			Min: float64(data.Min),
			Max: float64(data.Max),
		}
	}

	return stats, nil
}

// LoadScopedStats loads metric statistics for a job with scope-aware grouping.
//
// Similar to LoadStats but supports multiple scopes and returns statistics grouped
// by scope with hardware IDs (e.g., per-core, per-socket statistics).
//
// Parameters:
//   - job: Job metadata
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - ctx: Context (currently unused)
//
// Returns:
//   - ScopedJobStats: Map of metric → scope → []ScopedStats (with hostname and ID)
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadScopedStats(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
) (schema.ScopedJobStats, error) {
	queries, assignedScope, err := buildQueries(job, metrics, scopes, 0)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	scopedJobStats := make(schema.ScopedJobStats)

	for i, row := range resBody.Results {
		if len(row) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}
		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]

		if _, ok := scopedJobStats[metric]; !ok {
			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
		}

		if _, ok := scopedJobStats[metric][scope]; !ok {
			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)

			SanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
				Hostname: query.Hostname,
				ID:       id,
				Data: &schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
			})
		}

		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
		if len(scopedJobStats[metric][scope]) == 0 {
			delete(scopedJobStats[metric], scope)
			if len(scopedJobStats[metric]) == 0 {
				delete(scopedJobStats, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return scopedJobStats, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return scopedJobStats, nil
}

// LoadNodeData loads metric data for specific nodes in a cluster over a time range.
//
// Unlike LoadData which operates on job resources, this function queries arbitrary nodes
// directly. Useful for system monitoring and node status views.
//
// Parameters:
//   - cluster: Cluster name
//   - metrics: List of metric names
//   - nodes: List of node hostnames (nil = all nodes in cluster via ForAllNodes)
//   - scopes: Requested metric scopes (currently unused - always node scope)
//   - from, to: Time range
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of hostname → metric → []JobMetric
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadNodeData(
	cluster string,
	metrics, nodes []string,
	scopes []schema.MetricScope,
	from, to time.Time,
	ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
	req := APIQueryRequest{
		Cluster:   cluster,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	if nodes == nil {
		req.ForAllNodes = append(req.ForAllNodes, metrics...)
	} else {
		for _, node := range nodes {
			for _, metric := range metrics {
				req.Queries = append(req.Queries, APIQuery{
					Hostname:   node,
					Metric:     metric,
					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
				})
			}
		}
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]map[string][]*schema.JobMetric)
	for i, res := range resBody.Results {
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}

		var query APIQuery
		if resBody.Queries != nil {
			query = resBody.Queries[i]
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		qdata := res[0]
		if qdata.Error != nil {
			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
			continue
		}

		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
			continue
		}

		SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)

		hostdata, ok := data[query.Hostname]
		if !ok {
			hostdata = make(map[string][]*schema.JobMetric)
			data[query.Hostname] = hostdata
		}

		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
			Unit:     mc.Unit,
			Timestep: mc.Timestep,
			Series: []schema.Series{
				{
					Hostname: query.Hostname,
					Data:     qdata.Data,
					Statistics: schema.MetricStatistics{
						Avg: float64(qdata.Avg),
						Min: float64(qdata.Min),
						Max: float64(qdata.Max),
					},
				},
			},
		})
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// LoadNodeListData loads metric data for a list of nodes with full scope transformation support.
//
// This is the most flexible node data loading function, supporting arbitrary scopes and
// resolution. Uses buildNodeQueries for proper scope transformation based on topology.
//
// Parameters:
//   - cluster: Cluster name
//   - subCluster: SubCluster name (empty string to infer from node names)
//   - nodes: List of node hostnames
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - resolution: Data resolution in seconds
//   - from, to: Time range
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of hostname → JobData (metric → scope → JobMetric)
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadNodeListData(
	cluster, subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	from, to time.Time,
	ctx context.Context,
) (map[string]schema.JobData, error) {
	// Note: Order of node data is not guaranteed after this point
	queries, assignedScope, err := buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution))
	if err != nil {
		cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:   cluster,
		Queries:   queries,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]schema.JobData)

	// Add safety check for index out of range issues
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		var query APIQuery
		if resBody.Queries != nil {
			if i < len(resBody.Queries) {
				query = resBody.Queries[i]
			} else {
				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
					i, len(resBody.Queries))
				continue
			}
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
			continue
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = int(row[0].Resolution)
		}

		// Init Nested Map Data Structures If Not Found
		hostData, ok := data[query.Hostname]
		if !ok {
			hostData = make(schema.JobData)
			data[query.Hostname] = hostData
		}

		metricData, ok := hostData[metric]
		if !ok {
			metricData = make(map[schema.MetricScope]*schema.JobMetric)
			data[query.Hostname][metric] = metricData
		}

		scopeData, ok := metricData[scope]
		if !ok {
			scopeData = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			data[query.Hostname][metric][scope] = scopeData
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)

			SanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopeData.Series = append(scopeData.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// buildNodeQueries constructs APIQuery structures for node-based queries with scope transformation.
//
// Similar to buildQueries but operates on node lists rather than job resources.
// Supports dynamic subcluster lookup when subCluster parameter is empty.
//
// Parameters:
//   - cluster: Cluster name
//   - subCluster: SubCluster name (empty = infer from node hostnames)
//   - nodes: List of node hostnames
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - resolution: Data resolution in seconds
//
// Returns:
//   - []APIQuery: List of queries to execute
//   - []schema.MetricScope: Assigned scope for each query
//   - error: Returns error if topology lookup fails or unhandled scope combination
func buildNodeQueries(
	cluster string,
	subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int64,
) ([]APIQuery, []schema.MetricScope, error) {
	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(nodes))

	// Get Topol before loop if subCluster given
	var subClusterTopol *schema.SubCluster
	var scterr error
	if subCluster != "" {
		subClusterTopol, scterr = archive.GetSubCluster(cluster, subCluster)
		if scterr != nil {
			cclog.Errorf("could not load cluster %s subCluster %s topology: %s", cluster, subCluster, scterr.Error())
			return nil, nil, scterr
		}
	}

	for _, metric := range metrics {
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
			continue
		}

		// Skip if metric is removed for subcluster
		if mc.SubClusters != nil && IsMetricRemovedForSubCluster(mc, subCluster) {
			continue
		}

		// Avoid duplicates...
		handledScopes := make([]schema.MetricScope, 0, 3)

	nodeScopesLoop:
		for _, requestedScope := range scopes {
			nativeScope := mc.Scope

			scope := nativeScope.Max(requestedScope)
			for _, s := range handledScopes {
				if scope == s {
					continue nodeScopesLoop
				}
			}
			handledScopes = append(handledScopes, scope)

			for _, hostname := range nodes {

				// If no subCluster given, get it by node
				if subCluster == "" {
					subClusterName, scnerr := archive.GetSubClusterByNode(cluster, hostname)
					if scnerr != nil {
						return nil, nil, scnerr
					}
					subClusterTopol, scterr = archive.GetSubCluster(cluster, subClusterName)
					if scterr != nil {
						return nil, nil, scterr
					}
				}

				// Always full node hwthread id list, no partial queries expected
				topology := subClusterTopol.Topology
				acceleratorIds := topology.GetAcceleratorIDs()

				// Moved check here if metric matches hardware specs
				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
					continue
				}

				scopeResults, ok := BuildScopeQueries(
					nativeScope, requestedScope,
					metric, hostname,
					&topology, topology.Node, acceleratorIds,
				)

				if !ok {
					return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
				}

				for _, sr := range scopeResults {
					queries = append(queries, APIQuery{
						Metric:     sr.Metric,
						Hostname:   sr.Hostname,
						Aggregate:  sr.Aggregate,
						Type:       sr.Type,
						TypeIds:    sr.TypeIds,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, sr.Scope)
				}
			}
		}
	}

	return queries, assignedScope, nil
}


Now let me read the remaining files to get a complete picture of the package.