cc-backend/pkg/metricstore/query.go

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements high-level query functions for loading job metric data
// with automatic scope transformation and aggregation.
//
// Key Concepts:
//
// Metric Scopes: Metrics are collected at different granularities (native scope):
//   - HWThread: Per hardware thread
//   - Core: Per CPU core
//   - Socket: Per CPU socket
//   - MemoryDomain: Per memory domain (NUMA)
//   - Accelerator: Per GPU/accelerator
//   - Node: Per compute node
//
// Scope Transformation: The buildQueries functions transform between native scope
// and requested scope by:
//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
//   - Rejecting requests for finer granularity than available
//   - Handling special cases (e.g., Accelerator metrics)
//
// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
// based on cluster topology and job resources.
package metricstore

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

type InternalMetricStore struct{}

var MetricStoreHandle *InternalMetricStore

// HealthCheck delegates to the internal MemoryStore's HealthCheck.
func (ccms *InternalMetricStore) HealthCheck(cluster string,
	nodes []string, metrics []string,
) (map[string]HealthCheckResult, error) {
	return GetMemoryStore().HealthCheck(cluster, nodes, metrics)
}

// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
// When set to a non-nil function, LoadData will call this function instead of the default implementation.
var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int, resampleAlgo string) (schema.JobData, error)

// LoadData loads metric data for a specific job with automatic scope transformation.
//
// This is the primary function for retrieving job metric data. It handles:
//   - Building queries with scope transformation via buildQueries
//   - Fetching data from the metric store
//   - Organizing results by metric and scope
//   - Converting NaN statistics to 0 for JSON compatibility
//   - Partial error handling (returns data for successful queries even if some fail)
//
// Parameters:
//   - job: Job metadata including cluster, resources, and time range
//   - metrics: List of metric names to load
//   - scopes: Requested metric scopes (will be transformed to match native scopes)
//   - ctx: Context for cancellation (currently unused but reserved for future use)
//   - resolution: Data resolution in seconds (0 for native resolution)
//
// Returns:
//   - JobData: Map of metric → scope → JobMetric with time-series data and statistics
//   - Error: Returns error if query building or fetching fails, or partial error listing failed hosts
//
// Example:
//
//	jobData, err := LoadData(job, []string{"cpu_load", "mem_used"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 60)
func (ccms *InternalMetricStore) LoadData(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
	resolution int,
	resampleAlgo string,
) (schema.JobData, error) {
	if TestLoadDataCallback != nil {
		return TestLoadDataCallback(job, metrics, scopes, ctx, resolution, resampleAlgo)
	}

	queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return schema.JobData{}, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:      job.Cluster,
		From:         job.StartTime,
		To:           job.StartTime + int64(job.Duration),
		Queries:      queries,
		WithStats:    true,
		WithData:     true,
		ResampleAlgo: resampleAlgo,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return schema.JobData{}, err
	}

	var errors []string
	jobData := schema.JobData{Metrics: make(map[string]schema.ScopedMetrics)}

	// Add safety check for potential index out of range errors
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(job.Cluster, metric)

		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
			continue
		}

		if _, ok := jobData.Metrics[metric]; !ok {
			jobData.Metrics[metric] = make(schema.ScopedMetrics)
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = int(row[0].Resolution)
		}

		jobMetric, ok := jobData.Metrics[metric][scope]
		if !ok {
			jobMetric = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			jobData.Metrics[metric][scope] = jobMetric
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)

			SanitizeStats(&res.Avg, &res.Min, &res.Max)

			jobMetric.Series = append(jobMetric.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}

		// So that one can later check len(jobData.Metrics):
		if len(jobMetric.Series) == 0 {
			delete(jobData.Metrics[metric], scope)
			if len(jobData.Metrics[metric]) == 0 {
				delete(jobData.Metrics, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return jobData, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return jobData, nil
}

// buildQueries constructs APIQuery structures with automatic scope transformation for a job.
//
// This function implements the core scope transformation logic, handling all combinations of
// native metric scopes and requested scopes. It uses the cluster topology to determine which
// hardware IDs to include in each query.
//
// Scope Transformation Rules:
//   - If native scope >= requested scope: Aggregates data (Aggregate=true in APIQuery)
//   - If native scope < requested scope: Returns error (cannot increase granularity)
//   - Special handling for Accelerator scope (independent of CPU hierarchy)
//
// The function generates one or more APIQuery per (metric, scope, host) combination:
//   - For non-aggregated queries: One query with all relevant IDs
//   - For aggregated queries: May generate multiple queries (e.g., one per socket/core)
//
// Parameters:
//   - job: Job metadata including cluster, subcluster, and resource allocation
//   - metrics: List of metrics to query
//   - scopes: Requested scopes for each metric
//   - resolution: Data resolution in seconds
//
// Returns:
//   - []APIQuery: List of queries to execute
//   - []schema.MetricScope: Assigned scope for each query (after transformation)
//   - error: Returns error if topology lookup fails or unhandled scope combination encountered
func buildQueries(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int64,
) ([]APIQuery, []schema.MetricScope, error) {
	if len(job.Resources) == 0 {
		return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > no resources allocated for job %d", job.JobID)
	}

	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(job.Resources))

	subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster)
	if scerr != nil {
		return nil, nil, scerr
	}
	topology := subcluster.Topology

	for _, metric := range metrics {
		mc := archive.GetMetricConfig(job.Cluster, metric)
		if mc == nil {
			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
			continue
		}

		// Skip if metric is removed for subcluster
		if len(mc.SubClusters) != 0 && IsMetricRemovedForSubCluster(mc, job.SubCluster) {
			continue
		}

		// Avoid duplicates...
		handledScopes := make([]schema.MetricScope, 0, 3)

	scopesLoop:
		for _, requestedScope := range scopes {
			nativeScope := mc.Scope
			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
				continue
			}

			scope := nativeScope.Max(requestedScope)
			for _, s := range handledScopes {
				if scope == s {
					continue scopesLoop
				}
			}
			handledScopes = append(handledScopes, scope)

			for _, host := range job.Resources {
				hwthreads := host.HWThreads
				if hwthreads == nil {
					hwthreads = topology.Node
				}

				scopeResults, ok := BuildScopeQueries(
					nativeScope, requestedScope,
					metric, host.Hostname,
					&topology, hwthreads, host.Accelerators,
				)

				if !ok {
					return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
				}

				for _, sr := range scopeResults {
					queries = append(queries, APIQuery{
						Metric:     sr.Metric,
						Hostname:   sr.Hostname,
						Aggregate:  sr.Aggregate,
						Type:       sr.Type,
						TypeIds:    sr.TypeIds,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, sr.Scope)
				}
			}
		}
	}

	return queries, assignedScope, nil
}

// LoadStats loads only metric statistics (avg/min/max) for a job at node scope.
//
// This is an optimized version of LoadData that fetches only statistics without
// time-series data, reducing bandwidth and memory usage. Always queries at node scope.
//
// Parameters:
//   - job: Job metadata
//   - metrics: List of metric names
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of metric → hostname → statistics
//   - Error on query building or fetching failure
func (ccms *InternalMetricStore) LoadStats(
	job *schema.Job,
	metrics []string,
	ctx context.Context,
) (map[string]map[string]schema.MetricStatistics, error) {
	// TODO(#166): Add scope parameter for analysis view accelerator normalization
	queries, _, err := buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
	for i, res := range resBody.Results {
		if i >= len(req.Queries) {
			cclog.Warnf("LoadStats: result index %d exceeds queries length %d", i, len(req.Queries))
			break
		}
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}
		query := req.Queries[i]
		metric := query.Metric
		data := res[0]
		if data.Error != nil {
			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
			continue
		}

		metricdata, ok := stats[metric]
		if !ok {
			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
			stats[metric] = metricdata
		}

		if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() {
			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
			continue
		}

		metricdata[query.Hostname] = schema.MetricStatistics{
			Avg: float64(data.Avg),
			Min: float64(data.Min),
			Max: float64(data.Max),
		}
	}

	return stats, nil
}

// LoadScopedStats loads metric statistics for a job with scope-aware grouping.
//
// Similar to LoadStats but supports multiple scopes and returns statistics grouped
// by scope with hardware IDs (e.g., per-core, per-socket statistics).
//
// Parameters:
//   - job: Job metadata
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - ctx: Context (currently unused)
//
// Returns:
//   - ScopedJobStats: Map of metric → scope → []ScopedStats (with hostname and ID)
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadScopedStats(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
) (schema.ScopedJobStats, error) {
	queries, assignedScope, err := buildQueries(job, metrics, scopes, 0)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return schema.ScopedJobStats{}, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return schema.ScopedJobStats{}, err
	}

	var errors []string
	scopedJobStats := schema.ScopedJobStats{Metrics: make(map[string]schema.ScopedMetricStats)}

	for i, row := range resBody.Results {
		if len(row) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}
		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]

		if _, ok := scopedJobStats.Metrics[metric]; !ok {
			scopedJobStats.Metrics[metric] = make(schema.ScopedMetricStats)
		}

		if _, ok := scopedJobStats.Metrics[metric][scope]; !ok {
			scopedJobStats.Metrics[metric][scope] = make([]*schema.ScopedStats, 0)
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)

			SanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopedJobStats.Metrics[metric][scope] = append(scopedJobStats.Metrics[metric][scope], &schema.ScopedStats{
				Hostname: query.Hostname,
				ID:       id,
				Data: &schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
			})
		}

		// So that one can later check len(scopedJobStats.Metrics[metric][scope]): Remove from map if empty
		if len(scopedJobStats.Metrics[metric][scope]) == 0 {
			delete(scopedJobStats.Metrics[metric], scope)
			if len(scopedJobStats.Metrics[metric]) == 0 {
				delete(scopedJobStats.Metrics, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return scopedJobStats, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return scopedJobStats, nil
}

// LoadNodeData loads metric data for specific nodes in a cluster over a time range.
//
// Unlike LoadData which operates on job resources, this function queries arbitrary nodes
// directly. Useful for system monitoring and node status views.
//
// Parameters:
//   - cluster: Cluster name
//   - metrics: List of metric names
//   - nodes: List of node hostnames (nil = all nodes in cluster via ForAllNodes)
//   - scopes: Requested metric scopes (currently unused - always node scope)
//   - from, to: Time range
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of hostname → metric → []JobMetric
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadNodeData(
	cluster string,
	metrics, nodes []string,
	scopes []schema.MetricScope,
	from, to time.Time,
	ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
	req := APIQueryRequest{
		Cluster:   cluster,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	if nodes == nil {
		req.ForAllNodes = append(req.ForAllNodes, metrics...)
	} else {
		for _, node := range nodes {
			for _, metric := range metrics {
				req.Queries = append(req.Queries, APIQuery{
					Hostname:   node,
					Metric:     metric,
					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
				})
			}
		}
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]map[string][]*schema.JobMetric)
	for i, res := range resBody.Results {
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}

		var query APIQuery
		if resBody.Queries != nil {
			query = resBody.Queries[i]
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		qdata := res[0]
		if qdata.Error != nil {
			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
			continue
		}

		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
			continue
		}

		SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)

		hostdata, ok := data[query.Hostname]
		if !ok {
			hostdata = make(map[string][]*schema.JobMetric)
			data[query.Hostname] = hostdata
		}

		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
			Unit:     mc.Unit,
			Timestep: mc.Timestep,
			Series: []schema.Series{
				{
					Hostname: query.Hostname,
					Data:     qdata.Data,
					Statistics: schema.MetricStatistics{
						Avg: float64(qdata.Avg),
						Min: float64(qdata.Min),
						Max: float64(qdata.Max),
					},
				},
			},
		})
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// LoadNodeListData loads metric data for a list of nodes with full scope transformation support.
//
// This is the most flexible node data loading function, supporting arbitrary scopes and
// resolution. Uses buildNodeQueries for proper scope transformation based on topology.
//
// Parameters:
//   - cluster: Cluster name
//   - subCluster: SubCluster name (empty string to infer from node names)
//   - nodes: List of node hostnames
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - resolution: Data resolution in seconds
//   - from, to: Time range
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of hostname → JobData (metric → scope → JobMetric)
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadNodeListData(
	cluster, subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	from, to time.Time,
	ctx context.Context,
	resampleAlgo string,
) (map[string]schema.JobData, error) {
	// Note: Order of node data is not guaranteed after this point
	queries, assignedScope, err := buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution))
	if err != nil {
		cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:      cluster,
		Queries:      queries,
		From:         from.Unix(),
		To:           to.Unix(),
		WithStats:    true,
		WithData:     true,
		ResampleAlgo: resampleAlgo,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]schema.JobData)

	// Add safety check for index out of range issues
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		var query APIQuery
		if resBody.Queries != nil {
			if i < len(resBody.Queries) {
				query = resBody.Queries[i]
			} else {
				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
					i, len(resBody.Queries))
				continue
			}
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
			continue
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = int(row[0].Resolution)
		}

		// Init Nested Map Data Structures If Not Found
		hostData, ok := data[query.Hostname]
		if !ok {
			hostData = schema.JobData{Metrics: make(map[string]schema.ScopedMetrics)}
			data[query.Hostname] = hostData
		}

		metricData, ok := hostData.Metrics[metric]
		if !ok {
			metricData = make(schema.ScopedMetrics)
			hostData.Metrics[metric] = metricData
		}

		scopeData, ok := metricData[scope]
		if !ok {
			scopeData = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			metricData[scope] = scopeData
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)

			SanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopeData.Series = append(scopeData.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// buildNodeQueries constructs APIQuery structures for node-based queries with scope transformation.
//
// Similar to buildQueries but operates on node lists rather than job resources.
// Supports dynamic subcluster lookup when subCluster parameter is empty.
//
// Parameters:
//   - cluster: Cluster name
//   - subCluster: SubCluster name (empty = infer from node hostnames)
//   - nodes: List of node hostnames
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - resolution: Data resolution in seconds
//
// Returns:
//   - []APIQuery: List of queries to execute
//   - []schema.MetricScope: Assigned scope for each query
//   - error: Returns error if topology lookup fails or unhandled scope combination
func buildNodeQueries(
	cluster string,
	subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int64,
) ([]APIQuery, []schema.MetricScope, error) {
	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(nodes))

	// Get Topol before loop if subCluster given
	var subClusterTopol *schema.SubCluster
	var scterr error
	if subCluster != "" {
		subClusterTopol, scterr = archive.GetSubCluster(cluster, subCluster)
		if scterr != nil {
			cclog.Errorf("could not load cluster %s subCluster %s topology: %s", cluster, subCluster, scterr.Error())
			return nil, nil, scterr
		}
	}

	for _, metric := range metrics {
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
			continue
		}

		// Skip if metric is removed for subcluster
		if mc.SubClusters != nil && IsMetricRemovedForSubCluster(mc, subCluster) {
			continue
		}

		// Avoid duplicates...
		handledScopes := make([]schema.MetricScope, 0, 3)

	nodeScopesLoop:
		for _, requestedScope := range scopes {
			nativeScope := mc.Scope

			scope := nativeScope.Max(requestedScope)
			for _, s := range handledScopes {
				if scope == s {
					continue nodeScopesLoop
				}
			}
			handledScopes = append(handledScopes, scope)

			for _, hostname := range nodes {

				// If no subCluster given, get it by node
				if subCluster == "" {
					subClusterName, scnerr := archive.GetSubClusterByNode(cluster, hostname)
					if scnerr != nil {
						return nil, nil, scnerr
					}
					subClusterTopol, scterr = archive.GetSubCluster(cluster, subClusterName)
					if scterr != nil {
						return nil, nil, scterr
					}
				}

				// Always full node hwthread id list, no partial queries expected
				topology := subClusterTopol.Topology
				acceleratorIds := topology.GetAcceleratorIDs()

				// Moved check here if metric matches hardware specs
				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
					continue
				}

				scopeResults, ok := BuildScopeQueries(
					nativeScope, requestedScope,
					metric, hostname,
					&topology, topology.Node, acceleratorIds,
				)

				if !ok {
					return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
				}

				for _, sr := range scopeResults {
					queries = append(queries, APIQuery{
						Metric:     sr.Metric,
						Hostname:   sr.Hostname,
						Aggregate:  sr.Aggregate,
						Type:       sr.Type,
						TypeIds:    sr.TypeIds,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, sr.Scope)
				}
			}
		}
	}

	return queries, assignedScope, nil
}