   100→	}
   101→
   102→	req := APIQueryRequest{
   103→		Cluster:   job.Cluster,
   104→		From:      job.StartTime,
   105→		To:        job.StartTime + int64(job.Duration),
   106→		Queries:   queries,
   107→		WithStats: true,
   108→		WithData:  true,
   109→	}
   110→
   111→	resBody, err := FetchData(req)
   112→	if err != nil {
   113→		cclog.Errorf("Error while fetching data : %s", err.Error())
   114→		return nil, err
   115→	}
   116→
   117→	var errors []string
   118→	jobData := make(schema.JobData)
   119→
   120→	// Add safety check for potential index out of range errors
   121→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   122→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   123→			len(req.Queries), len(resBody.Results), len(assignedScope))
   124→		if len(resBody.Results) > len(req.Queries) {
   125→			resBody.Results = resBody.Results[:len(req.Queries)]
   126→		}
   127→		if len(assignedScope) > len(req.Queries) {
   128→			assignedScope = assignedScope[:len(req.Queries)]
   129→		}
   130→	}
   131→
   132→	for i, row := range resBody.Results {
   133→		// Safety check to prevent index out of range errors
   134→		if i >= len(req.Queries) || i >= len(assignedScope) {
   135→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   136→				i, len(req.Queries), len(assignedScope))
   137→			continue
   138→		}
   139→
   140→		query := req.Queries[i]
   141→		metric := query.Metric
   142→		scope := assignedScope[i]
   143→		mc := archive.GetMetricConfig(job.Cluster, metric)
   144→
   145→		if mc == nil {
   146→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   147→			continue
   148→		}
   149→
   150→		if _, ok := jobData[metric]; !ok {
   151→			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
   152→		}
   153→
   154→		res := mc.Timestep
   155→		if len(row) > 0 {
   156→			res = int(row[0].Resolution)
   157→		}
   158→
   159→		jobMetric, ok := jobData[metric][scope]
   160→		if !ok {
   161→			jobMetric = &schema.JobMetric{
   162→				Unit:     mc.Unit,
   163→				Timestep: res,
   164→				Series:   make([]schema.Series, 0),
   165→			}
   166→			jobData[metric][scope] = jobMetric
   167→		}
   168→
   169→		for ndx, res := range row {
   170→			if res.Error != nil {
   171→				/* Build list for "partial errors", if any */
   172→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   173→				continue
   174→			}
   175→
   176→			id := (*string)(nil)
   177→			if query.Type != nil {
   178→				// Check if ndx is within the bounds of TypeIds slice
   179→				if ndx < len(query.TypeIds) {
   180→					id = new(string)
   181→					*id = query.TypeIds[ndx]
   182→				} else {
   183→					// Log the error but continue processing
   184→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   185→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   186→				}
   187→			}
   188→
   189→			sanitizeStats(&res)
   190→
   191→			jobMetric.Series = append(jobMetric.Series, schema.Series{
   192→				Hostname: query.Hostname,
   193→				ID:       id,
   194→				Statistics: schema.MetricStatistics{
   195→					Avg: float64(res.Avg),
   196→					Min: float64(res.Min),
   197→					Max: float64(res.Max),
   198→				},
   199→				Data: res.Data,
   200→			})
   201→		}
   202→
   203→		// So that one can later check len(jobData):
   204→		if len(jobMetric.Series) == 0 {
   205→			delete(jobData[metric], scope)
   206→			if len(jobData[metric]) == 0 {
   207→				delete(jobData, metric)
   208→			}
   209→		}
   210→	}
   211→
   212→	if len(errors) != 0 {
   213→		/* Returns list for "partial errors" */
   214→		return jobData, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   215→	}
   216→	return jobData, nil
   217→}
   218→
   219→// Pre-converted scope strings avoid repeated string(MetricScope) allocations during
   220→// query construction. These are used in APIQuery.Type field throughout buildQueries
   221→// and buildNodeQueries functions. Converting once at package initialization improves
   222→// performance for high-volume query building.
   223→var (
   224→	hwthreadString     = string(schema.MetricScopeHWThread)
   225→	coreString         = string(schema.MetricScopeCore)
   226→	memoryDomainString = string(schema.MetricScopeMemoryDomain)
   227→	socketString       = string(schema.MetricScopeSocket)
   228→	acceleratorString  = string(schema.MetricScopeAccelerator)
   229→)
   230→
   231→// buildQueries constructs APIQuery structures with automatic scope transformation for a job.
   232→//
   233→// This function implements the core scope transformation logic, handling all combinations of
   234→// native metric scopes and requested scopes. It uses the cluster topology to determine which
   235→// hardware IDs to include in each query.
   236→//
   237→// Scope Transformation Rules:
   238→//   - If native scope >= requested scope: Aggregates data (Aggregate=true in APIQuery)
   239→//   - If native scope < requested scope: Returns error (cannot increase granularity)
   240→//   - Special handling for Accelerator scope (independent of CPU hierarchy)
   241→//
   242→// The function generates one or more APIQuery per (metric, scope, host) combination:
   243→//   - For non-aggregated queries: One query with all relevant IDs
   244→//   - For aggregated queries: May generate multiple queries (e.g., one per socket/core)
   245→//
   246→// Parameters:
   247→//   - job: Job metadata including cluster, subcluster, and resource allocation
   248→//   - metrics: List of metrics to query
   249→//   - scopes: Requested scopes for each metric
   250→//   - resolution: Data resolution in seconds
   251→//
   252→// Returns:
   253→//   - []APIQuery: List of queries to execute
   254→//   - []schema.MetricScope: Assigned scope for each query (after transformation)
   255→//   - error: Returns error if topology lookup fails or unhandled scope combination encountered
   256→func buildQueries(
   257→	job *schema.Job,
   258→	metrics []string,
   259→	scopes []schema.MetricScope,
   260→	resolution int64,
   261→) ([]APIQuery, []schema.MetricScope, error) {
   262→	if len(job.Resources) == 0 {
   263→		return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > no resources allocated for job %d", job.JobID)
   264→	}
   265→
   266→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
   267→	assignedScope := []schema.MetricScope{}
   268→
   269→	subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster)
   270→	if scerr != nil {
   271→		return nil, nil, scerr
   272→	}
   273→	topology := subcluster.Topology
   274→
   275→	for _, metric := range metrics {
   276→		mc := archive.GetMetricConfig(job.Cluster, metric)
   277→		if mc == nil {
   278→			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
   279→			continue

	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  true,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	jobData := make(schema.JobData)

	// Add safety check for potential index out of range errors
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		// Safety check to prevent index out of range errors
		if i >= len(req.Queries) || i >= len(assignedScope) {
			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
				i, len(req.Queries), len(assignedScope))
			continue
		}

		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(job.Cluster, metric)

		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
			continue
		}

		if _, ok := jobData[metric]; !ok {
			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = int(row[0].Resolution)
		}

		jobMetric, ok := jobData[metric][scope]
		if !ok {
			jobMetric = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			jobData[metric][scope] = jobMetric
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			sanitizeStats(&res)

			jobMetric.Series = append(jobMetric.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}

		// So that one can later check len(jobData):
		if len(jobMetric.Series) == 0 {
			delete(jobData[metric], scope)
			if len(jobData[metric]) == 0 {
				delete(jobData, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return jobData, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return jobData, nil
}

// Pre-converted scope strings avoid repeated string(MetricScope) allocations during
// query construction. These are used in APIQuery.Type field throughout buildQueries
// and buildNodeQueries functions. Converting once at package initialization improves
// performance for high-volume query building.
var (
	hwthreadString     = string(schema.MetricScopeHWThread)
	coreString         = string(schema.MetricScopeCore)
	memoryDomainString = string(schema.MetricScopeMemoryDomain)
	socketString       = string(schema.MetricScopeSocket)
	acceleratorString  = string(schema.MetricScopeAccelerator)
)

// buildQueries constructs APIQuery structures with automatic scope transformation for a job.
//
// This function implements the core scope transformation logic, handling all combinations of
// native metric scopes and requested scopes. It uses the cluster topology to determine which
// hardware IDs to include in each query.
//
// Scope Transformation Rules:
//   - If native scope >= requested scope: Aggregates data (Aggregate=true in APIQuery)
//   - If native scope < requested scope: Returns error (cannot increase granularity)
//   - Special handling for Accelerator scope (independent of CPU hierarchy)
//
// The function generates one or more APIQuery per (metric, scope, host) combination:
//   - For non-aggregated queries: One query with all relevant IDs
//   - For aggregated queries: May generate multiple queries (e.g., one per socket/core)
//
// Parameters:
//   - job: Job metadata including cluster, subcluster, and resource allocation
//   - metrics: List of metrics to query
//   - scopes: Requested scopes for each metric
//   - resolution: Data resolution in seconds
//
// Returns:
//   - []APIQuery: List of queries to execute
//   - []schema.MetricScope: Assigned scope for each query (after transformation)
//   - error: Returns error if topology lookup fails or unhandled scope combination encountered
func buildQueries(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int64,
) ([]APIQuery, []schema.MetricScope, error) {
	if len(job.Resources) == 0 {
		return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > no resources allocated for job %d", job.JobID)
	}

	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
	assignedScope := []schema.MetricScope{}

	subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster)
	if scerr != nil {
		return nil, nil, scerr
	}
	topology := subcluster.Topology

	for _, metric := range metrics {
		mc := archive.GetMetricConfig(job.Cluster, metric)
		if mc == nil {
			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
			continue


   580→	}
   581→
   582→	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
   583→	for i, res := range resBody.Results {
   584→		if len(res) == 0 {
   585→			// No Data Found For Metric, Logged in FetchData to Warn
   586→			continue
   587→		}
   588→		query := req.Queries[i]
   589→		metric := query.Metric
   590→		data := res[0]
   591→		if data.Error != nil {
   592→			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
   593→			continue
   594→		}
   595→
   596→		metricdata, ok := stats[metric]
   597→		if !ok {
   598→			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
   599→			stats[metric] = metricdata
   600→		}
   601→
   602→		if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() {
   603→			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
   604→			continue
   605→		}
   606→
   607→		metricdata[query.Hostname] = schema.MetricStatistics{
   608→			Avg: float64(data.Avg),
   609→			Min: float64(data.Min),
   610→			Max: float64(data.Max),
   611→		}
   612→	}
   613→
   614→	return stats, nil
   615→}
   616→
   617→// LoadScopedStats loads metric statistics for a job with scope-aware grouping.
   618→//
   619→// Similar to LoadStats but supports multiple scopes and returns statistics grouped
   620→// by scope with hardware IDs (e.g., per-core, per-socket statistics).
   621→//
   622→// Parameters:
   623→//   - job: Job metadata
   624→//   - metrics: List of metric names
   625→//   - scopes: Requested metric scopes
   626→//   - ctx: Context (currently unused)
   627→//
   628→// Returns:
   629→//   - ScopedJobStats: Map of metric → scope → []ScopedStats (with hostname and ID)
   630→//   - Error or partial error listing failed queries
   631→func (ccms *InternalMetricStore) LoadScopedStats(
   632→	job *schema.Job,
   633→	metrics []string,
   634→	scopes []schema.MetricScope,
   635→	ctx context.Context,
   636→) (schema.ScopedJobStats, error) {
   637→	queries, assignedScope, err := buildQueries(job, metrics, scopes, 0)
   638→	if err != nil {
   639→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   640→		return nil, err
   641→	}
   642→
   643→	req := APIQueryRequest{
   644→		Cluster:   job.Cluster,
   645→		From:      job.StartTime,
   646→		To:        job.StartTime + int64(job.Duration),
   647→		Queries:   queries,
   648→		WithStats: true,
   649→		WithData:  false,
   650→	}
   651→
   652→	resBody, err := FetchData(req)
   653→	if err != nil {
   654→		cclog.Errorf("Error while fetching data : %s", err.Error())
   655→		return nil, err
   656→	}
   657→
   658→	var errors []string
   659→	scopedJobStats := make(schema.ScopedJobStats)
   660→
   661→	for i, row := range resBody.Results {
   662→		if len(row) == 0 {
   663→			// No Data Found For Metric, Logged in FetchData to Warn
   664→			continue
   665→		}
   666→		query := req.Queries[i]
   667→		metric := query.Metric
   668→		scope := assignedScope[i]
   669→
   670→		if _, ok := scopedJobStats[metric]; !ok {
   671→			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
   672→		}
   673→
   674→		if _, ok := scopedJobStats[metric][scope]; !ok {
   675→			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
   676→		}
   677→
   678→		for ndx, res := range row {
   679→			if res.Error != nil {
   680→				/* Build list for "partial errors", if any */
   681→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   682→				continue
   683→			}
   684→
   685→			id := (*string)(nil)
   686→			if query.Type != nil {
   687→				// Check if ndx is within the bounds of TypeIds slice
   688→				if ndx < len(query.TypeIds) {
   689→					id = new(string)
   690→					*id = query.TypeIds[ndx]
   691→				} else {
   692→					// Log the error but continue processing
   693→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   694→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   695→				}
   696→			}
   697→
   698→			sanitizeStats(&res)
   699→
   700→			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
   701→				Hostname: query.Hostname,
   702→				ID:       id,
   703→				Data: &schema.MetricStatistics{
   704→					Avg: float64(res.Avg),
   705→					Min: float64(res.Min),
   706→					Max: float64(res.Max),
   707→				},
   708→			})
   709→		}
   710→
   711→		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
   712→		if len(scopedJobStats[metric][scope]) == 0 {
   713→			delete(scopedJobStats[metric], scope)
   714→			if len(scopedJobStats[metric]) == 0 {
   715→				delete(scopedJobStats, metric)
   716→			}
   717→		}
   718→	}
   719→
   720→	if len(errors) != 0 {
   721→		/* Returns list for "partial errors" */
   722→		return scopedJobStats, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   723→	}
   724→	return scopedJobStats, nil
   725→}
   726→
   727→// LoadNodeData loads metric data for specific nodes in a cluster over a time range.
   728→//
   729→// Unlike LoadData which operates on job resources, this function queries arbitrary nodes
   730→// directly. Useful for system monitoring and node status views.
   731→//
   732→// Parameters:
   733→//   - cluster: Cluster name
   734→//   - metrics: List of metric names
   735→//   - nodes: List of node hostnames (nil = all nodes in cluster via ForAllNodes)
   736→//   - scopes: Requested metric scopes (currently unused - always node scope)
   737→//   - from, to: Time range
   738→//   - ctx: Context (currently unused)
   739→//
   740→// Returns:
   741→//   - Map of hostname → metric → []JobMetric
   742→//   - Error or partial error listing failed queries
   743→func (ccms *InternalMetricStore) LoadNodeData(
   744→	cluster string,
   745→	metrics, nodes []string,
   746→	scopes []schema.MetricScope,
   747→	from, to time.Time,
   748→	ctx context.Context,
   749→) (map[string]map[string][]*schema.JobMetric, error) {
   750→	req := APIQueryRequest{
   751→		Cluster:   cluster,
   752→		From:      from.Unix(),
   753→		To:        to.Unix(),
   754→		WithStats: true,
   755→		WithData:  true,
   756→	}
   757→
   758→	if nodes == nil {
   759→		req.ForAllNodes = append(req.ForAllNodes, metrics...)
   760→	} else {
   761→		for _, node := range nodes {
   762→			for _, metric := range metrics {
   763→				req.Queries = append(req.Queries, APIQuery{
   764→					Hostname:   node,
   765→					Metric:     metric,
   766→					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
   767→				})
   768→			}
   769→		}
   770→	}
   771→
   772→	resBody, err := FetchData(req)
   773→	if err != nil {
   774→		cclog.Errorf("Error while fetching data : %s", err.Error())
   775→		return nil, err
   776→	}
   777→
   778→	var errors []string
   779→	data := make(map[string]map[string][]*schema.JobMetric)

	}

	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
	for i, res := range resBody.Results {
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}
		query := req.Queries[i]
		metric := query.Metric
		data := res[0]
		if data.Error != nil {
			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
			continue
		}

		metricdata, ok := stats[metric]
		if !ok {
			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
			stats[metric] = metricdata
		}

		if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() {
			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
			continue
		}

		metricdata[query.Hostname] = schema.MetricStatistics{
			Avg: float64(data.Avg),
			Min: float64(data.Min),
			Max: float64(data.Max),
		}
	}

	return stats, nil
}

// LoadScopedStats loads metric statistics for a job with scope-aware grouping.
//
// Similar to LoadStats but supports multiple scopes and returns statistics grouped
// by scope with hardware IDs (e.g., per-core, per-socket statistics).
//
// Parameters:
//   - job: Job metadata
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - ctx: Context (currently unused)
//
// Returns:
//   - ScopedJobStats: Map of metric → scope → []ScopedStats (with hostname and ID)
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadScopedStats(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
) (schema.ScopedJobStats, error) {
	queries, assignedScope, err := buildQueries(job, metrics, scopes, 0)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	scopedJobStats := make(schema.ScopedJobStats)

	for i, row := range resBody.Results {
		if len(row) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}
		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]

		if _, ok := scopedJobStats[metric]; !ok {
			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
		}

		if _, ok := scopedJobStats[metric][scope]; !ok {
			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			sanitizeStats(&res)

			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
				Hostname: query.Hostname,
				ID:       id,
				Data: &schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
			})
		}

		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
		if len(scopedJobStats[metric][scope]) == 0 {
			delete(scopedJobStats[metric], scope)
			if len(scopedJobStats[metric]) == 0 {
				delete(scopedJobStats, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return scopedJobStats, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return scopedJobStats, nil
}

// LoadNodeData loads metric data for specific nodes in a cluster over a time range.
//
// Unlike LoadData which operates on job resources, this function queries arbitrary nodes
// directly. Useful for system monitoring and node status views.
//
// Parameters:
//   - cluster: Cluster name
//   - metrics: List of metric names
//   - nodes: List of node hostnames (nil = all nodes in cluster via ForAllNodes)
//   - scopes: Requested metric scopes (currently unused - always node scope)
//   - from, to: Time range
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of hostname → metric → []JobMetric
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadNodeData(
	cluster string,
	metrics, nodes []string,
	scopes []schema.MetricScope,
	from, to time.Time,
	ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
	req := APIQueryRequest{
		Cluster:   cluster,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	if nodes == nil {
		req.ForAllNodes = append(req.ForAllNodes, metrics...)
	} else {
		for _, node := range nodes {
			for _, metric := range metrics {
				req.Queries = append(req.Queries, APIQuery{
					Hostname:   node,
					Metric:     metric,
					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
				})
			}
		}
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]map[string][]*schema.JobMetric)


   780→	for i, res := range resBody.Results {
   781→		if len(res) == 0 {
   782→			// No Data Found For Metric, Logged in FetchData to Warn
   783→			continue
   784→		}
   785→
   786→		var query APIQuery
   787→		if resBody.Queries != nil {
   788→			query = resBody.Queries[i]
   789→		} else {
   790→			query = req.Queries[i]
   791→		}
   792→
   793→		metric := query.Metric
   794→		qdata := res[0]
   795→		if qdata.Error != nil {
   796→			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
   797→		}
   798→
   799→		sanitizeStats(&qdata)
   800→
   801→		hostdata, ok := data[query.Hostname]
   802→		if !ok {
   803→			hostdata = make(map[string][]*schema.JobMetric)
   804→			data[query.Hostname] = hostdata
   805→		}
   806→
   807→		mc := archive.GetMetricConfig(cluster, metric)
   808→		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
   809→			Unit:     mc.Unit,
   810→			Timestep: mc.Timestep,
   811→			Series: []schema.Series{
   812→				{
   813→					Hostname: query.Hostname,
   814→					Data:     qdata.Data,
   815→					Statistics: schema.MetricStatistics{
   816→						Avg: float64(qdata.Avg),
   817→						Min: float64(qdata.Min),
   818→						Max: float64(qdata.Max),
   819→					},
   820→				},
   821→			},
   822→		})
   823→	}
   824→
   825→	if len(errors) != 0 {
   826→		/* Returns list of "partial errors" */
   827→		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   828→	}
   829→
   830→	return data, nil
   831→}
   832→
   833→// LoadNodeListData loads metric data for a list of nodes with full scope transformation support.
   834→//
   835→// This is the most flexible node data loading function, supporting arbitrary scopes and
   836→// resolution. Uses buildNodeQueries for proper scope transformation based on topology.
   837→//
   838→// Parameters:
   839→//   - cluster: Cluster name
   840→//   - subCluster: SubCluster name (empty string to infer from node names)
   841→//   - nodes: List of node hostnames
   842→//   - metrics: List of metric names
   843→//   - scopes: Requested metric scopes
   844→//   - resolution: Data resolution in seconds
   845→//   - from, to: Time range
   846→//   - ctx: Context (currently unused)
   847→//
   848→// Returns:
   849→//   - Map of hostname → JobData (metric → scope → JobMetric)
   850→//   - Error or partial error listing failed queries
   851→func (ccms *InternalMetricStore) LoadNodeListData(
   852→	cluster, subCluster string,
   853→	nodes []string,
   854→	metrics []string,
   855→	scopes []schema.MetricScope,
   856→	resolution int,
   857→	from, to time.Time,
   858→	ctx context.Context,
   859→) (map[string]schema.JobData, error) {
   860→	// Note: Order of node data is not guaranteed after this point
   861→	queries, assignedScope, err := buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution))
   862→	if err != nil {
   863→		cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
   864→		return nil, err
   865→	}
   866→
   867→	// Verify assignment is correct - log any inconsistencies for debugging
   868→	if len(queries) != len(assignedScope) {
   869→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
   870→			len(queries), len(assignedScope))
   871→	}
   872→
   873→	req := APIQueryRequest{
   874→		Cluster:   cluster,
   875→		Queries:   queries,
   876→		From:      from.Unix(),
   877→		To:        to.Unix(),
   878→		WithStats: true,
   879→		WithData:  true,
   880→	}
   881→
   882→	resBody, err := FetchData(req)
   883→	if err != nil {
   884→		cclog.Errorf("Error while fetching data : %s", err.Error())
   885→		return nil, err
   886→	}
   887→
   888→	var errors []string
   889→	data := make(map[string]schema.JobData)
   890→
   891→	// Add safety check for index out of range issues
   892→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   893→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   894→			len(req.Queries), len(resBody.Results), len(assignedScope))
   895→		if len(resBody.Results) > len(req.Queries) {
   896→			resBody.Results = resBody.Results[:len(req.Queries)]
   897→		}
   898→		if len(assignedScope) > len(req.Queries) {
   899→			assignedScope = assignedScope[:len(req.Queries)]
   900→		}
   901→	}
   902→
   903→	for i, row := range resBody.Results {
   904→		// Safety check to prevent index out of range errors
   905→		if i >= len(req.Queries) || i >= len(assignedScope) {
   906→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   907→				i, len(req.Queries), len(assignedScope))
   908→			continue
   909→		}
   910→
   911→		var query APIQuery
   912→		if resBody.Queries != nil {
   913→			if i < len(resBody.Queries) {
   914→				query = resBody.Queries[i]
   915→			} else {
   916→				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
   917→					i, len(resBody.Queries))
   918→				continue
   919→			}
   920→		} else {
   921→			query = req.Queries[i]
   922→		}
   923→
   924→		metric := query.Metric
   925→		scope := assignedScope[i]
   926→		mc := archive.GetMetricConfig(cluster, metric)
   927→		if mc == nil {
   928→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   929→			continue
   930→		}
   931→
   932→		res := mc.Timestep
   933→		if len(row) > 0 {
   934→			res = int(row[0].Resolution)
   935→		}
   936→
   937→		// Init Nested Map Data Structures If Not Found
   938→		hostData, ok := data[query.Hostname]
   939→		if !ok {
   940→			hostData = make(schema.JobData)
   941→			data[query.Hostname] = hostData
   942→		}
   943→
   944→		metricData, ok := hostData[metric]
   945→		if !ok {
   946→			metricData = make(map[schema.MetricScope]*schema.JobMetric)
   947→			data[query.Hostname][metric] = metricData
   948→		}
   949→
   950→		scopeData, ok := metricData[scope]
   951→		if !ok {
   952→			scopeData = &schema.JobMetric{
   953→				Unit:     mc.Unit,
   954→				Timestep: res,
   955→				Series:   make([]schema.Series, 0),
   956→			}
   957→			data[query.Hostname][metric][scope] = scopeData
   958→		}
   959→
   960→		for ndx, res := range row {
   961→			if res.Error != nil {
   962→				/* Build list for "partial errors", if any */
   963→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   964→				continue
   965→			}
   966→
   967→			id := (*string)(nil)
   968→			if query.Type != nil {
   969→				// Check if ndx is within the bounds of TypeIds slice
   970→				if ndx < len(query.TypeIds) {
   971→					id = new(string)
   972→					*id = query.TypeIds[ndx]
   973→				} else {
   974→					// Log the error but continue processing
   975→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   976→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   977→				}
   978→			}
   979→
   980→			sanitizeStats(&res)
   981→
   982→			scopeData.Series = append(scopeData.Series, schema.Series{
   983→				Hostname: query.Hostname,
   984→				ID:       id,
   985→				Statistics: schema.MetricStatistics{
   986→					Avg: float64(res.Avg),
   987→					Min: float64(res.Min),
   988→					Max: float64(res.Max),
   989→				},
   990→				Data: res.Data,
   991→			})
   992→		}
   993→	}
   994→
   995→	if len(errors) != 0 {
   996→		/* Returns list of "partial errors" */
   997→		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   998→	}
   999→
  1000→	return data, nil
  1001→}
  1002→
  1003→// buildNodeQueries constructs APIQuery structures for node-based queries with scope transformation.
  1004→//
  1005→// Similar to buildQueries but operates on node lists rather than job resources.
  1006→// Supports dynamic subcluster lookup when subCluster parameter is empty.
  1007→//
  1008→// Parameters:
  1009→//   - cluster: Cluster name
  1010→//   - subCluster: SubCluster name (empty = infer from node hostnames)
  1011→//   - nodes: List of node hostnames
  1012→//   - metrics: List of metric names
  1013→//   - scopes: Requested metric scopes
  1014→//   - resolution: Data resolution in seconds
  1015→//
  1016→// Returns:
  1017→//   - []APIQuery: List of queries to execute
  1018→//   - []schema.MetricScope: Assigned scope for each query
  1019→//   - error: Returns error if topology lookup fails or unhandled scope combination
  1020→func buildNodeQueries(
  1021→	cluster string,
  1022→	subCluster string,
  1023→	nodes []string,
  1024→	metrics []string,
  1025→	scopes []schema.MetricScope,
  1026→	resolution int64,
  1027→) ([]APIQuery, []schema.MetricScope, error) {
  1028→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
  1029→	assignedScope := []schema.MetricScope{}
  1030→
  1031→	// Get Topol before loop if subCluster given
  1032→	var subClusterTopol *schema.SubCluster
  1033→	var scterr error
  1034→	if subCluster != "" {
  1035→		subClusterTopol, scterr = archive.GetSubCluster(cluster, subCluster)
  1036→		if scterr != nil {
  1037→			cclog.Errorf("could not load cluster %s subCluster %s topology: %s", cluster, subCluster, scterr.Error())
  1038→			return nil, nil, scterr
  1039→		}
  1040→	}
  1041→
  1042→	for _, metric := range metrics {
  1043→		mc := archive.GetMetricConfig(cluster, metric)
  1044→		if mc == nil {
  1045→			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
  1046→			continue
  1047→		}
  1048→
  1049→		// Skip if metric is removed for subcluster
  1050→		if mc.SubClusters != nil {
  1051→			isRemoved := false
  1052→			for _, scConfig := range mc.SubClusters {
  1053→				if scConfig.Name == subCluster && scConfig.Remove {
  1054→					isRemoved = true
  1055→					break
  1056→				}
  1057→			}
  1058→			if isRemoved {
  1059→				continue
  1060→			}
  1061→		}
  1062→
  1063→		// Avoid duplicates using map for O(1) lookup
  1064→		handledScopes := make(map[schema.MetricScope]bool, 3)
  1065→
  1066→		for _, requestedScope := range scopes {
  1067→			nativeScope := mc.Scope
  1068→
  1069→			scope := nativeScope.Max(requestedScope)
  1070→			if handledScopes[scope] {
  1071→				continue
  1072→			}
  1073→			handledScopes[scope] = true
  1074→
  1075→			for _, hostname := range nodes {
  1076→
  1077→				// If no subCluster given, get it by node
  1078→				if subCluster == "" {
  1079→					subClusterName, scnerr := archive.GetSubClusterByNode(cluster, hostname)
  1080→					if scnerr != nil {
  1081→						return nil, nil, scnerr
  1082→					}
  1083→					subClusterTopol, scterr = archive.GetSubCluster(cluster, subClusterName)
  1084→					if scterr != nil {
  1085→						return nil, nil, scterr
  1086→					}
  1087→				}
  1088→
  1089→				// Always full node hwthread id list, no partial queries expected -> Use "topology.Node" directly where applicable
  1090→				// Always full accelerator id list, no partial queries expected -> Use "acceleratorIds" directly where applicable
  1091→				topology := subClusterTopol.Topology
  1092→				acceleratorIds := topology.GetAcceleratorIDs()
  1093→
  1094→				// Moved check here if metric matches hardware specs
  1095→				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
  1096→					continue
  1097→				}
  1098→
  1099→				// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
  1100→				if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
  1101→					if scope != schema.MetricScopeAccelerator {
  1102→						// Skip all other catched cases
  1103→						continue
  1104→					}
  1105→
  1106→					queries = append(queries, APIQuery{
  1107→						Metric:     metric,
  1108→						Hostname:   hostname,
  1109→						Aggregate:  false,
  1110→						Type:       &acceleratorString,
  1111→						TypeIds:    acceleratorIds,
  1112→						Resolution: resolution,
  1113→					})
  1114→					assignedScope = append(assignedScope, schema.MetricScopeAccelerator)
  1115→					continue
  1116→				}
  1117→
  1118→				// Accelerator -> Node
  1119→				if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
  1120→					if len(acceleratorIds) == 0 {
  1121→						continue
  1122→					}
  1123→
  1124→					queries = append(queries, APIQuery{
  1125→						Metric:     metric,
  1126→						Hostname:   hostname,
  1127→						Aggregate:  true,
  1128→						Type:       &acceleratorString,
  1129→						TypeIds:    acceleratorIds,
  1130→						Resolution: resolution,
  1131→					})
  1132→					assignedScope = append(assignedScope, scope)
  1133→					continue
  1134→				}
  1135→
  1136→				// HWThread -> HWThread
  1137→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
  1138→					queries = append(queries, APIQuery{
  1139→						Metric:     metric,
  1140→						Hostname:   hostname,
  1141→						Aggregate:  false,
  1142→						Type:       &hwthreadString,
  1143→						TypeIds:    intToStringSlice(topology.Node),
  1144→						Resolution: resolution,
  1145→					})
  1146→					assignedScope = append(assignedScope, scope)
  1147→					continue
  1148→				}
  1149→
  1150→				// HWThread -> Core
  1151→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
  1152→					cores, _ := topology.GetCoresFromHWThreads(topology.Node)
  1153→					for _, core := range cores {
  1154→						queries = append(queries, APIQuery{
  1155→							Metric:     metric,
  1156→							Hostname:   hostname,
  1157→							Aggregate:  true,
  1158→							Type:       &hwthreadString,
  1159→							TypeIds:    intToStringSlice(topology.Core[core]),
  1160→							Resolution: resolution,
  1161→						})
  1162→						assignedScope = append(assignedScope, scope)
  1163→					}
  1164→					continue
  1165→				}
  1166→
  1167→				// HWThread -> Socket
  1168→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
  1169→					sockets, _ := topology.GetSocketsFromHWThreads(topology.Node)
  1170→					for _, socket := range sockets {
  1171→						queries = append(queries, APIQuery{
  1172→							Metric:     metric,
  1173→							Hostname:   hostname,
  1174→							Aggregate:  true,
  1175→							Type:       &hwthreadString,
  1176→							TypeIds:    intToStringSlice(topology.Socket[socket]),
  1177→							Resolution: resolution,
  1178→						})
  1179→						assignedScope = append(assignedScope, scope)
  1180→					}
  1181→					continue
  1182→				}
  1183→
  1184→				// HWThread -> Node
  1185→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
  1186→					queries = append(queries, APIQuery{
  1187→						Metric:     metric,
  1188→						Hostname:   hostname,
  1189→						Aggregate:  true,
  1190→						Type:       &hwthreadString,
  1191→						TypeIds:    intToStringSlice(topology.Node),
  1192→						Resolution: resolution,
  1193→					})
  1194→					assignedScope = append(assignedScope, scope)
  1195→					continue
  1196→				}
  1197→
  1198→				// Core -> Core
  1199→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
  1200→					cores, _ := topology.GetCoresFromHWThreads(topology.Node)
  1201→					queries = append(queries, APIQuery{
  1202→						Metric:     metric,
  1203→						Hostname:   hostname,
  1204→						Aggregate:  false,
  1205→						Type:       &coreString,
  1206→						TypeIds:    intToStringSlice(cores),
  1207→						Resolution: resolution,
  1208→					})
  1209→					assignedScope = append(assignedScope, scope)
  1210→					continue
  1211→				}
  1212→
  1213→				// Core -> Socket
  1214→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
  1215→					sockets, _ := topology.GetSocketsFromCores(topology.Node)
  1216→					for _, socket := range sockets {
  1217→						queries = append(queries, APIQuery{
  1218→							Metric:     metric,
  1219→							Hostname:   hostname,
  1220→							Aggregate:  true,
  1221→							Type:       &coreString,
  1222→							TypeIds:    intToStringSlice(topology.Socket[socket]),
  1223→							Resolution: resolution,
  1224→						})
  1225→						assignedScope = append(assignedScope, scope)
  1226→					}
  1227→					continue
  1228→				}
  1229→
  1230→				// Core -> Node
  1231→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
  1232→					cores, _ := topology.GetCoresFromHWThreads(topology.Node)
  1233→					queries = append(queries, APIQuery{
  1234→						Metric:     metric,
  1235→						Hostname:   hostname,
  1236→						Aggregate:  true,
  1237→						Type:       &coreString,
  1238→						TypeIds:    intToStringSlice(cores),
  1239→						Resolution: resolution,
  1240→					})
  1241→					assignedScope = append(assignedScope, scope)
  1242→					continue
  1243→				}
  1244→
  1245→				// MemoryDomain -> MemoryDomain
  1246→				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
  1247→					sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node)
  1248→					queries = append(queries, APIQuery{
  1249→						Metric:     metric,
  1250→						Hostname:   hostname,
  1251→						Aggregate:  false,
  1252→						Type:       &memoryDomainString,
  1253→						TypeIds:    intToStringSlice(sockets),
  1254→						Resolution: resolution,
  1255→					})
  1256→					assignedScope = append(assignedScope, scope)
  1257→					continue
  1258→				}
  1259→
  1260→				// MemoryDomain -> Node
  1261→				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
  1262→					sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node)
  1263→					queries = append(queries, APIQuery{
  1264→						Metric:     metric,
  1265→						Hostname:   hostname,
  1266→						Aggregate:  true,
  1267→						Type:       &memoryDomainString,
  1268→						TypeIds:    intToStringSlice(sockets),
  1269→						Resolution: resolution,
  1270→					})
  1271→					assignedScope = append(assignedScope, scope)
  1272→					continue
  1273→				}
  1274→
  1275→				// Socket -> Socket
  1276→				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
  1277→					sockets, _ := topology.GetSocketsFromHWThreads(topology.Node)
  1278→					queries = append(queries, APIQuery{
  1279→						Metric:     metric,
  1280→						Hostname:   hostname,
  1281→						Aggregate:  false,
  1282→						Type:       &socketString,
  1283→						TypeIds:    intToStringSlice(sockets),
  1284→						Resolution: resolution,
  1285→					})
  1286→					assignedScope = append(assignedScope, scope)
  1287→					continue
  1288→				}
  1289→
  1290→				// Socket -> Node
  1291→				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
  1292→					sockets, _ := topology.GetSocketsFromHWThreads(topology.Node)
  1293→					queries = append(queries, APIQuery{
  1294→						Metric:     metric,
  1295→						Hostname:   hostname,
  1296→						Aggregate:  true,
  1297→						Type:       &socketString,
  1298→						TypeIds:    intToStringSlice(sockets),
  1299→						Resolution: resolution,
  1300→					})
  1301→					assignedScope = append(assignedScope, scope)
  1302→					continue
  1303→				}
  1304→
  1305→				// Node -> Node
  1306→				if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
  1307→					queries = append(queries, APIQuery{
  1308→						Metric:     metric,
  1309→						Hostname:   hostname,
  1310→						Resolution: resolution,
  1311→					})
  1312→					assignedScope = append(assignedScope, scope)
  1313→					continue
  1314→				}
  1315→
  1316→				return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
  1317→			}
  1318→		}
  1319→	}
  1320→
  1321→	return queries, assignedScope, nil
  1322→}
  1323→
  1324→// sanitizeStats converts NaN statistics to zero for JSON compatibility.
  1325→//
  1326→// schema.Float with NaN values cannot be properly JSON-encoded, so we convert
  1327→// NaN to 0. This loses the distinction between "no data" and "zero value",
  1328→// but maintains API compatibility.
  1329→func sanitizeStats(data *APIMetricData) {
  1330→	if data.Avg.IsNaN() {
  1331→		data.Avg = schema.Float(0)
  1332→	}
  1333→	if data.Min.IsNaN() {
  1334→		data.Min = schema.Float(0)
  1335→	}
  1336→	if data.Max.IsNaN() {
  1337→		data.Max = schema.Float(0)
  1338→	}
  1339→}
  1340→
  1341→// intToStringSlice converts a slice of integers to a slice of strings.
  1342→// Used to convert hardware thread/core/socket IDs from topology (int) to APIQuery TypeIds (string).
  1343→//
  1344→// Optimized to reuse a byte buffer for string conversion, reducing allocations.
  1345→func intToStringSlice(is []int) []string {
  1346→	if len(is) == 0 {
  1347→		return nil
  1348→	}
  1349→
  1350→	ss := make([]string, len(is))
  1351→	buf := make([]byte, 0, 16) // Reusable buffer for integer conversion
  1352→	for i, x := range is {
  1353→		buf = strconv.AppendInt(buf[:0], int64(x), 10)
  1354→		ss[i] = string(buf)
  1355→	}
  1356→	return ss
  1357→}
  1358→

	for i, res := range resBody.Results {
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}

		var query APIQuery
		if resBody.Queries != nil {
			query = resBody.Queries[i]
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		qdata := res[0]
		if qdata.Error != nil {
			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
		}

		sanitizeStats(&qdata)

		hostdata, ok := data[query.Hostname]
		if !ok {
			hostdata = make(map[string][]*schema.JobMetric)
			data[query.Hostname] = hostdata
		}

		mc := archive.GetMetricConfig(cluster, metric)
		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
			Unit:     mc.Unit,
			Timestep: mc.Timestep,
			Series: []schema.Series{
				{
					Hostname: query.Hostname,
					Data:     qdata.Data,
					Statistics: schema.MetricStatistics{
						Avg: float64(qdata.Avg),
						Min: float64(qdata.Min),
						Max: float64(qdata.Max),
					},
				},
			},
		})
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// LoadNodeListData loads metric data for a list of nodes with full scope transformation support.
//
// This is the most flexible node data loading function, supporting arbitrary scopes and
// resolution. Uses buildNodeQueries for proper scope transformation based on topology.
//
// Parameters:
//   - cluster: Cluster name
//   - subCluster: SubCluster name (empty string to infer from node names)
//   - nodes: List of node hostnames
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - resolution: Data resolution in seconds
//   - from, to: Time range
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of hostname → JobData (metric → scope → JobMetric)
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadNodeListData(
	cluster, subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	from, to time.Time,
	ctx context.Context,
) (map[string]schema.JobData, error) {
	// Note: Order of node data is not guaranteed after this point
	queries, assignedScope, err := buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution))
	if err != nil {
		cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:   cluster,
		Queries:   queries,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]schema.JobData)

	// Add safety check for index out of range issues
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		// Safety check to prevent index out of range errors
		if i >= len(req.Queries) || i >= len(assignedScope) {
			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
				i, len(req.Queries), len(assignedScope))
			continue
		}

		var query APIQuery
		if resBody.Queries != nil {
			if i < len(resBody.Queries) {
				query = resBody.Queries[i]
			} else {
				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
					i, len(resBody.Queries))
				continue
			}
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
			continue
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = int(row[0].Resolution)
		}

		// Init Nested Map Data Structures If Not Found
		hostData, ok := data[query.Hostname]
		if !ok {
			hostData = make(schema.JobData)
			data[query.Hostname] = hostData
		}

		metricData, ok := hostData[metric]
		if !ok {
			metricData = make(map[schema.MetricScope]*schema.JobMetric)
			data[query.Hostname][metric] = metricData
		}

		scopeData, ok := metricData[scope]
		if !ok {
			scopeData = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			data[query.Hostname][metric][scope] = scopeData
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			sanitizeStats(&res)

			scopeData.Series = append(scopeData.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// buildNodeQueries constructs APIQuery structures for node-based queries with scope transformation.
//
// Similar to buildQueries but operates on node lists rather than job resources.
// Supports dynamic subcluster lookup when subCluster parameter is empty.
//
// Parameters:
//   - cluster: Cluster name
//   - subCluster: SubCluster name (empty = infer from node hostnames)
//   - nodes: List of node hostnames
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - resolution: Data resolution in seconds
//
// Returns:
//   - []APIQuery: List of queries to execute
//   - []schema.MetricScope: Assigned scope for each query
//   - error: Returns error if topology lookup fails or unhandled scope combination
func buildNodeQueries(
	cluster string,
	subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int64,
) ([]APIQuery, []schema.MetricScope, error) {
	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
	assignedScope := []schema.MetricScope{}

	// Get Topol before loop if subCluster given
	var subClusterTopol *schema.SubCluster
	var scterr error
	if subCluster != "" {
		subClusterTopol, scterr = archive.GetSubCluster(cluster, subCluster)
		if scterr != nil {
			cclog.Errorf("could not load cluster %s subCluster %s topology: %s", cluster, subCluster, scterr.Error())
			return nil, nil, scterr
		}
	}

	for _, metric := range metrics {
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
			continue
		}

		// Skip if metric is removed for subcluster
		if mc.SubClusters != nil {
			isRemoved := false
			for _, scConfig := range mc.SubClusters {
				if scConfig.Name == subCluster && scConfig.Remove {
					isRemoved = true
					break
				}
			}
			if isRemoved {
				continue
			}
		}

		// Avoid duplicates using map for O(1) lookup
		handledScopes := make(map[schema.MetricScope]bool, 3)

		for _, requestedScope := range scopes {
			nativeScope := mc.Scope

			scope := nativeScope.Max(requestedScope)
			if handledScopes[scope] {
				continue
			}
			handledScopes[scope] = true

			for _, hostname := range nodes {

				// If no subCluster given, get it by node
				if subCluster == "" {
					subClusterName, scnerr := archive.GetSubClusterByNode(cluster, hostname)
					if scnerr != nil {
						return nil, nil, scnerr
					}
					subClusterTopol, scterr = archive.GetSubCluster(cluster, subClusterName)
					if scterr != nil {
						return nil, nil, scterr
					}
				}

				// Always full node hwthread id list, no partial queries expected -> Use "topology.Node" directly where applicable
				// Always full accelerator id list, no partial queries expected -> Use "acceleratorIds" directly where applicable
				topology := subClusterTopol.Topology
				acceleratorIds := topology.GetAcceleratorIDs()

				// Moved check here if metric matches hardware specs
				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
					continue
				}

				// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
				if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
					if scope != schema.MetricScopeAccelerator {
						// Skip all other catched cases
						continue
					}

					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   hostname,
						Aggregate:  false,
						Type:       &acceleratorString,
						TypeIds:    acceleratorIds,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, schema.MetricScopeAccelerator)
					continue
				}

				// Accelerator -> Node
				if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
					if len(acceleratorIds) == 0 {
						continue
					}

					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   hostname,
						Aggregate:  true,
						Type:       &acceleratorString,
						TypeIds:    acceleratorIds,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// HWThread -> HWThread
				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   hostname,
						Aggregate:  false,
						Type:       &hwthreadString,
						TypeIds:    intToStringSlice(topology.Node),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// HWThread -> Core
				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
					cores, _ := topology.GetCoresFromHWThreads(topology.Node)
					for _, core := range cores {
						queries = append(queries, APIQuery{
							Metric:     metric,
							Hostname:   hostname,
							Aggregate:  true,
							Type:       &hwthreadString,
							TypeIds:    intToStringSlice(topology.Core[core]),
							Resolution: resolution,
						})
						assignedScope = append(assignedScope, scope)
					}
					continue
				}

				// HWThread -> Socket
				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
					sockets, _ := topology.GetSocketsFromHWThreads(topology.Node)
					for _, socket := range sockets {
						queries = append(queries, APIQuery{
							Metric:     metric,
							Hostname:   hostname,
							Aggregate:  true,
							Type:       &hwthreadString,
							TypeIds:    intToStringSlice(topology.Socket[socket]),
							Resolution: resolution,
						})
						assignedScope = append(assignedScope, scope)
					}
					continue
				}

				// HWThread -> Node
				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   hostname,
						Aggregate:  true,
						Type:       &hwthreadString,
						TypeIds:    intToStringSlice(topology.Node),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// Core -> Core
				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
					cores, _ := topology.GetCoresFromHWThreads(topology.Node)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   hostname,
						Aggregate:  false,
						Type:       &coreString,
						TypeIds:    intToStringSlice(cores),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// Core -> Socket
				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
					sockets, _ := topology.GetSocketsFromCores(topology.Node)
					for _, socket := range sockets {
						queries = append(queries, APIQuery{
							Metric:     metric,
							Hostname:   hostname,
							Aggregate:  true,
							Type:       &coreString,
							TypeIds:    intToStringSlice(topology.Socket[socket]),
							Resolution: resolution,
						})
						assignedScope = append(assignedScope, scope)
					}
					continue
				}

				// Core -> Node
				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
					cores, _ := topology.GetCoresFromHWThreads(topology.Node)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   hostname,
						Aggregate:  true,
						Type:       &coreString,
						TypeIds:    intToStringSlice(cores),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// MemoryDomain -> MemoryDomain
				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
					sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   hostname,
						Aggregate:  false,
						Type:       &memoryDomainString,
						TypeIds:    intToStringSlice(sockets),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// MemoryDomain -> Node
				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
					sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   hostname,
						Aggregate:  true,
						Type:       &memoryDomainString,
						TypeIds:    intToStringSlice(sockets),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// Socket -> Socket
				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
					sockets, _ := topology.GetSocketsFromHWThreads(topology.Node)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   hostname,
						Aggregate:  false,
						Type:       &socketString,
						TypeIds:    intToStringSlice(sockets),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// Socket -> Node
				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
					sockets, _ := topology.GetSocketsFromHWThreads(topology.Node)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   hostname,
						Aggregate:  true,
						Type:       &socketString,
						TypeIds:    intToStringSlice(sockets),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// Node -> Node
				if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   hostname,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
			}
		}
	}

	return queries, assignedScope, nil
}

// sanitizeStats converts NaN statistics to zero for JSON compatibility.
//
// schema.Float with NaN values cannot be properly JSON-encoded, so we convert
// NaN to 0. This loses the distinction between "no data" and "zero value",
// but maintains API compatibility.
func sanitizeStats(data *APIMetricData) {
	if data.Avg.IsNaN() {
		data.Avg = schema.Float(0)
	}
	if data.Min.IsNaN() {
		data.Min = schema.Float(0)
	}
	if data.Max.IsNaN() {
		data.Max = schema.Float(0)
	}
}

// intToStringSlice converts a slice of integers to a slice of strings.
// Used to convert hardware thread/core/socket IDs from topology (int) to APIQuery TypeIds (string).
//
// Optimized to reuse a byte buffer for string conversion, reducing allocations.
func intToStringSlice(is []int) []string {
	if len(is) == 0 {
		return nil
	}

	ss := make([]string, len(is))
	buf := make([]byte, 0, 16) // Reusable buffer for integer conversion
	for i, x := range is {
		buf = strconv.AppendInt(buf[:0], int64(x), 10)
		ss[i] = string(buf)
	}
	return ss
}


Now I have a complete picture. Let me create the tasks and start implementing.

Key observations:
1. metricstore's `sanitizeStats` takes `*APIMetricData`, while metricstoreclient's takes `*schema.Float, *schema.Float, *schema.Float`
2. metricstore's `sanitizeStats` zeros each field independently; metricstoreclient zeros ALL if ANY is NaN
3. metricstore's `intToStringSlice` has the optimized buffer-reuse version
4. metricstore is missing MemoryDomain → Socket transformation
5. metricstore's `APIQuery.Resolution` is `int64`, metricstoreclient's is `int`

6. Now I'm starting with the shared `scopequery.go` file to establish the common foundation.


Now I have a full understanding of the code. Let me create task tracking and start implementing.


TaskCreate

Create pkg/metricstore/scopequery.go with shared types and functions

New file with ScopeQueryResult, BuildScopeQueries, IntToStringSlice, SanitizeStats, and scope string constants


Task #1 created successfully: Create pkg/metricstore/scopequery.go with shared types and functions


Update pkg/metricstore/query.go to use shared functions

Replace inline if-chains in buildQueries/buildNodeQueries with BuildScopeQueries calls, remove duplicated helpers


Task #2 created successfully: Update pkg/metricstore/query.go to use shared functions


Update metricstoreclient to use shared functions

Replace local buildScopeQueries, intToStringSlice, sanitizeStats with shared versions


Task #3 created successfully: Update metricstoreclient to use shared functions


Add tests and verify compilation

Create scopequery_test.go, run go build and go test


Task #4 created successfully: Add tests and verify compilation

Add tests and verify compilation


Updated task #1 status


Write

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file contains shared scope transformation logic used by both the internal
// metric store (pkg/metricstore) and the external cc-metric-store client
// (internal/metricstoreclient). It extracts the common algorithm for mapping
// between native metric scopes and requested scopes based on cluster topology.
package metricstore

import (
	"strconv"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Pre-converted scope strings avoid repeated string(MetricScope) allocations
// during query construction. Used in ScopeQueryResult.Type field.
var (
	HWThreadString     = string(schema.MetricScopeHWThread)
	CoreString         = string(schema.MetricScopeCore)
	MemoryDomainString = string(schema.MetricScopeMemoryDomain)
	SocketString       = string(schema.MetricScopeSocket)
	AcceleratorString  = string(schema.MetricScopeAccelerator)
)

// ScopeQueryResult is a package-independent intermediate type returned by
// BuildScopeQueries. Each consumer converts it to their own APIQuery type
// (adding Resolution and any other package-specific fields).
type ScopeQueryResult struct {
	Type      *string
	Metric    string
	Hostname  string
	TypeIds   []string
	Scope     schema.MetricScope
	Aggregate bool
}

// BuildScopeQueries generates scope query results for a given scope transformation.
// It returns a slice of results and a boolean indicating success.
// An empty slice means an expected exception (skip this combination).
// ok=false means an unhandled case (caller should return an error).
func BuildScopeQueries(
	nativeScope, requestedScope schema.MetricScope,
	metric, hostname string,
	topology *schema.Topology,
	hwthreads []int,
	accelerators []string,
) ([]ScopeQueryResult, bool) {
	scope := nativeScope.Max(requestedScope)
	results := []ScopeQueryResult{}

	hwthreadsStr := IntToStringSlice(hwthreads)

	// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
	if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
		if scope != schema.MetricScopeAccelerator {
			// Expected Exception -> Return Empty Slice
			return results, true
		}

		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &AcceleratorString,
			TypeIds:   accelerators,
			Scope:     schema.MetricScopeAccelerator,
		})
		return results, true
	}

	// Accelerator -> Node
	if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
		if len(accelerators) == 0 {
			// Expected Exception -> Return Empty Slice
			return results, true
		}

		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &AcceleratorString,
			TypeIds:   accelerators,
			Scope:     scope,
		})
		return results, true
	}

	// HWThread -> HWThread
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &HWThreadString,
			TypeIds:   hwthreadsStr,
			Scope:     scope,
		})
		return results, true
	}

	// HWThread -> Core
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		for _, core := range cores {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &HWThreadString,
				TypeIds:   IntToStringSlice(topology.Core[core]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// HWThread -> Socket
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		for _, socket := range sockets {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &HWThreadString,
				TypeIds:   IntToStringSlice(topology.Socket[socket]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// HWThread -> Node
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &HWThreadString,
			TypeIds:   hwthreadsStr,
			Scope:     scope,
		})
		return results, true
	}

	// Core -> Core
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &CoreString,
			TypeIds:   IntToStringSlice(cores),
			Scope:     scope,
		})
		return results, true
	}

	// Core -> Socket
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromCores(hwthreads)
		for _, socket := range sockets {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &CoreString,
				TypeIds:   IntToStringSlice(topology.Socket[socket]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// Core -> Node
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &CoreString,
			TypeIds:   IntToStringSlice(cores),
			Scope:     scope,
		})
		return results, true
	}

	// MemoryDomain -> MemoryDomain
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &MemoryDomainString,
			TypeIds:   IntToStringSlice(memDomains),
			Scope:     scope,
		})
		return results, true
	}

	// MemoryDomain -> Socket
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains)
		if err != nil {
			cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err)
			// Rare Error Case -> Still Continue -> Return Empty Slice
			return results, true
		}

		// Create a query for each socket
		for _, domains := range socketToDomains {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &MemoryDomainString,
				TypeIds:   IntToStringSlice(domains),
				Scope:     scope,
			})
		}
		return results, true
	}

	// MemoryDomain -> Node
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &MemoryDomainString,
			TypeIds:   IntToStringSlice(memDomains),
			Scope:     scope,
		})
		return results, true
	}

	// Socket -> Socket
	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &SocketString,
			TypeIds:   IntToStringSlice(sockets),
			Scope:     scope,
		})
		return results, true
	}

	// Socket -> Node
	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &SocketString,
			TypeIds:   IntToStringSlice(sockets),
			Scope:     scope,
		})
		return results, true
	}

	// Node -> Node
	if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
		results = append(results, ScopeQueryResult{
			Metric:   metric,
			Hostname: hostname,
			Scope:    scope,
		})
		return results, true
	}

	// Unhandled Case
	return nil, false
}

// IntToStringSlice converts a slice of integers to a slice of strings.
// Used to convert hardware thread/core/socket IDs from topology (int) to query TypeIds (string).
// Optimized to reuse a byte buffer for string conversion, reducing allocations.
func IntToStringSlice(is []int) []string {
	if len(is) == 0 {
		return nil
	}

	ss := make([]string, len(is))
	buf := make([]byte, 0, 16) // Reusable buffer for integer conversion
	for i, x := range is {
		buf = strconv.AppendInt(buf[:0], int64(x), 10)
		ss[i] = string(buf)
	}
	return ss
}

// SanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
// If ANY of avg/min/max is NaN, ALL three are zeroed for consistency.
func SanitizeStats(avg, min, max *schema.Float) {
	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
		*avg = schema.Float(0)
		*min = schema.Float(0)
		*max = schema.Float(0)
	}
}


File created successfully at: /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/scopequery.go

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file contains shared scope transformation logic used by both the internal
// metric store (pkg/metricstore) and the external cc-metric-store client
// (internal/metricstoreclient). It extracts the common algorithm for mapping
// between native metric scopes and requested scopes based on cluster topology.
package metricstore

import (
	"strconv"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Pre-converted scope strings avoid repeated string(MetricScope) allocations
// during query construction. Used in ScopeQueryResult.Type field.
var (
	HWThreadString     = string(schema.MetricScopeHWThread)
	CoreString         = string(schema.MetricScopeCore)
	MemoryDomainString = string(schema.MetricScopeMemoryDomain)
	SocketString       = string(schema.MetricScopeSocket)
	AcceleratorString  = string(schema.MetricScopeAccelerator)
)

// ScopeQueryResult is a package-independent intermediate type returned by
// BuildScopeQueries. Each consumer converts it to their own APIQuery type
// (adding Resolution and any other package-specific fields).
type ScopeQueryResult struct {
	Type      *string
	Metric    string
	Hostname  string
	TypeIds   []string
	Scope     schema.MetricScope
	Aggregate bool
}

// BuildScopeQueries generates scope query results for a given scope transformation.
// It returns a slice of results and a boolean indicating success.
// An empty slice means an expected exception (skip this combination).
// ok=false means an unhandled case (caller should return an error).
func BuildScopeQueries(
	nativeScope, requestedScope schema.MetricScope,
	metric, hostname string,
	topology *schema.Topology,
	hwthreads []int,
	accelerators []string,
) ([]ScopeQueryResult, bool) {
	scope := nativeScope.Max(requestedScope)
	results := []ScopeQueryResult{}

	hwthreadsStr := IntToStringSlice(hwthreads)

	// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
	if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
		if scope != schema.MetricScopeAccelerator {
			// Expected Exception -> Return Empty Slice
			return results, true
		}

		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &AcceleratorString,
			TypeIds:   accelerators,
			Scope:     schema.MetricScopeAccelerator,
		})
		return results, true
	}

	// Accelerator -> Node
	if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
		if len(accelerators) == 0 {
			// Expected Exception -> Return Empty Slice
			return results, true
		}

		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &AcceleratorString,
			TypeIds:   accelerators,
			Scope:     scope,
		})
		return results, true
	}

	// HWThread -> HWThread
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &HWThreadString,
			TypeIds:   hwthreadsStr,
			Scope:     scope,
		})
		return results, true
	}

	// HWThread -> Core
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		for _, core := range cores {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &HWThreadString,
				TypeIds:   IntToStringSlice(topology.Core[core]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// HWThread -> Socket
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		for _, socket := range sockets {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &HWThreadString,
				TypeIds:   IntToStringSlice(topology.Socket[socket]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// HWThread -> Node
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &HWThreadString,
			TypeIds:   hwthreadsStr,
			Scope:     scope,
		})
		return results, true
	}

	// Core -> Core
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &CoreString,
			TypeIds:   IntToStringSlice(cores),
			Scope:     scope,
		})
		return results, true
	}

	// Core -> Socket
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromCores(hwthreads)
		for _, socket := range sockets {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &CoreString,
				TypeIds:   IntToStringSlice(topology.Socket[socket]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// Core -> Node
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &CoreString,
			TypeIds:   IntToStringSlice(cores),
			Scope:     scope,
		})
		return results, true
	}

	// MemoryDomain -> MemoryDomain
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &MemoryDomainString,
			TypeIds:   IntToStringSlice(memDomains),
			Scope:     scope,
		})
		return results, true
	}

	// MemoryDomain -> Socket
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains)
		if err != nil {
			cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err)
			// Rare Error Case -> Still Continue -> Return Empty Slice
			return results, true
		}

		// Create a query for each socket
		for _, domains := range socketToDomains {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &MemoryDomainString,
				TypeIds:   IntToStringSlice(domains),
				Scope:     scope,
			})
		}
		return results, true
	}

	// MemoryDomain -> Node
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &MemoryDomainString,
			TypeIds:   IntToStringSlice(memDomains),
			Scope:     scope,
		})
		return results, true
	}

	// Socket -> Socket
	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &SocketString,
			TypeIds:   IntToStringSlice(sockets),
			Scope:     scope,
		})
		return results, true
	}

	// Socket -> Node
	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &SocketString,
			TypeIds:   IntToStringSlice(sockets),
			Scope:     scope,
		})
		return results, true
	}

	// Node -> Node
	if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
		results = append(results, ScopeQueryResult{
			Metric:   metric,
			Hostname: hostname,
			Scope:    scope,
		})
		return results, true
	}

	// Unhandled Case
	return nil, false
}

// IntToStringSlice converts a slice of integers to a slice of strings.
// Used to convert hardware thread/core/socket IDs from topology (int) to query TypeIds (string).
// Optimized to reuse a byte buffer for string conversion, reducing allocations.
func IntToStringSlice(is []int) []string {
	if len(is) == 0 {
		return nil
	}

	ss := make([]string, len(is))
	buf := make([]byte, 0, 16) // Reusable buffer for integer conversion
	for i, x := range is {
		buf = strconv.AppendInt(buf[:0], int64(x), 10)
		ss[i] = string(buf)
	}
	return ss
}

// SanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
// If ANY of avg/min/max is NaN, ALL three are zeroed for consistency.
func SanitizeStats(avg, min, max *schema.Float) {
	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
		*avg = schema.Float(0)
		*min = schema.Float(0)
		*max = schema.Float(0)
	}
}


Updated task #2 status


Now update `pkg/metricstore/query.go` — replace the inline if-chains in both `buildQueries` and `buildNodeQueries`, remove duplicated helpers, and update `sanitizeStats` calls.


Edit

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)