Consolidate code for external and internal ccms buildQueries function

Entire-Checkpoint: fc3be444ef4c

internal/metricstoreclient/cc-metric-store-queries.go

@@ -37,23 +37,13 @@ package metricstoreclient
 
 import (
 	"fmt"
-	"strconv"
 
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
+	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
 	"github.com/ClusterCockpit/cc-lib/v2/schema"
 )
 
-// Scope string constants used in API queries.
-// Pre-converted to avoid repeated allocations during query building.
-var (
-	hwthreadString     = string(schema.MetricScopeHWThread)
-	coreString         = string(schema.MetricScopeCore)
-	memoryDomainString = string(schema.MetricScopeMemoryDomain)
-	socketString       = string(schema.MetricScopeSocket)
-	acceleratorString  = string(schema.MetricScopeAccelerator)
-)
-
 // buildQueries constructs API queries for job-specific metric data.
 // It iterates through metrics, scopes, and job resources to build the complete query set.
 //
@@ -126,21 +116,27 @@ func (ccms *CCMetricStore) buildQueries(
 					hwthreads = topology.Node
 				}
 
-				// Note: Expected exceptions will return as empty slices -> Continue
-				hostQueries, hostScopes := buildScopeQueries(
+				scopeResults, ok := metricstore.BuildScopeQueries(
 					nativeScope, requestedScope,
 					remoteName, host.Hostname,
 					topology, hwthreads, host.Accelerators,
-					resolution,
 				)
 
-				// Note: Unexpected errors, such as unhandled cases, will return as nils -> Error
-				if hostQueries == nil && hostScopes == nil {
+				if !ok {
 					return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
 				}
 
-				queries = append(queries, hostQueries...)
-				assignedScope = append(assignedScope, hostScopes...)
+				for _, sr := range scopeResults {
+					queries = append(queries, APIQuery{
+						Metric:     sr.Metric,
+						Hostname:   sr.Hostname,
+						Aggregate:  sr.Aggregate,
+						Type:       sr.Type,
+						TypeIds:    sr.TypeIds,
+						Resolution: resolution,
+					})
+					assignedScope = append(assignedScope, sr.Scope)
+				}
 			}
 		}
 	}
@@ -231,19 +227,27 @@ func (ccms *CCMetricStore) buildNodeQueries(
 					continue scopesLoop
 				}
 
-				nodeQueries, nodeScopes := buildScopeQueries(
+				scopeResults, ok := metricstore.BuildScopeQueries(
 					nativeScope, requestedScope,
 					remoteName, hostname,
 					topology, topology.Node, acceleratorIds,
-					resolution,
 				)
 
-				if len(nodeQueries) == 0 && len(nodeScopes) == 0 {
+				if !ok {
 					return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
 				}
 
-				queries = append(queries, nodeQueries...)
-				assignedScope = append(assignedScope, nodeScopes...)
+				for _, sr := range scopeResults {
+					queries = append(queries, APIQuery{
+						Metric:     sr.Metric,
+						Hostname:   sr.Hostname,
+						Aggregate:  sr.Aggregate,
+						Type:       sr.Type,
+						TypeIds:    sr.TypeIds,
+						Resolution: resolution,
+					})
+					assignedScope = append(assignedScope, sr.Scope)
+				}
 			}
 		}
 	}
@@ -251,277 +255,3 @@ func (ccms *CCMetricStore) buildNodeQueries(
 	return queries, assignedScope, nil
 }
 
-// buildScopeQueries generates API queries for a given scope transformation.
-// It returns a slice of queries and corresponding assigned scopes.
-// Some transformations (e.g., HWThread -> Core/Socket) may generate multiple queries.
-func buildScopeQueries(
-	nativeScope, requestedScope schema.MetricScope,
-	metric, hostname string,
-	topology *schema.Topology,
-	hwthreads []int,
-	accelerators []string,
-	resolution int,
-) ([]APIQuery, []schema.MetricScope) {
-	scope := nativeScope.Max(requestedScope)
-	queries := []APIQuery{}
-	scopes := []schema.MetricScope{}
-
-	hwthreadsStr := intToStringSlice(hwthreads)
-
-	// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
-	if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
-		if scope != schema.MetricScopeAccelerator {
-			// Expected Exception -> Continue -> Return Empty Slices
-			return queries, scopes
-		}
-
-		queries = append(queries, APIQuery{
-			Metric:     metric,
-			Hostname:   hostname,
-			Aggregate:  false,
-			Type:       &acceleratorString,
-			TypeIds:    accelerators,
-			Resolution: resolution,
-		})
-		scopes = append(scopes, schema.MetricScopeAccelerator)
-		return queries, scopes
-	}
-
-	// Accelerator -> Node
-	if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
-		if len(accelerators) == 0 {
-			// Expected Exception -> Continue -> Return Empty Slices
-			return queries, scopes
-		}
-
-		queries = append(queries, APIQuery{
-			Metric:     metric,
-			Hostname:   hostname,
-			Aggregate:  true,
-			Type:       &acceleratorString,
-			TypeIds:    accelerators,
-			Resolution: resolution,
-		})
-		scopes = append(scopes, scope)
-		return queries, scopes
-	}
-
-	// HWThread -> HWThread
-	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
-		queries = append(queries, APIQuery{
-			Metric:     metric,
-			Hostname:   hostname,
-			Aggregate:  false,
-			Type:       &hwthreadString,
-			TypeIds:    hwthreadsStr,
-			Resolution: resolution,
-		})
-		scopes = append(scopes, scope)
-		return queries, scopes
-	}
-
-	// HWThread -> Core
-	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
-		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
-		for _, core := range cores {
-			queries = append(queries, APIQuery{
-				Metric:     metric,
-				Hostname:   hostname,
-				Aggregate:  true,
-				Type:       &hwthreadString,
-				TypeIds:    intToStringSlice(topology.Core[core]),
-				Resolution: resolution,
-			})
-			scopes = append(scopes, scope)
-		}
-		return queries, scopes
-	}
-
-	// HWThread -> Socket
-	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
-		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
-		for _, socket := range sockets {
-			queries = append(queries, APIQuery{
-				Metric:     metric,
-				Hostname:   hostname,
-				Aggregate:  true,
-				Type:       &hwthreadString,
-				TypeIds:    intToStringSlice(topology.Socket[socket]),
-				Resolution: resolution,
-			})
-			scopes = append(scopes, scope)
-		}
-		return queries, scopes
-	}
-
-	// HWThread -> Node
-	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
-		queries = append(queries, APIQuery{
-			Metric:     metric,
-			Hostname:   hostname,
-			Aggregate:  true,
-			Type:       &hwthreadString,
-			TypeIds:    hwthreadsStr,
-			Resolution: resolution,
-		})
-		scopes = append(scopes, scope)
-		return queries, scopes
-	}
-
-	// Core -> Core
-	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
-		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
-		queries = append(queries, APIQuery{
-			Metric:     metric,
-			Hostname:   hostname,
-			Aggregate:  false,
-			Type:       &coreString,
-			TypeIds:    intToStringSlice(cores),
-			Resolution: resolution,
-		})
-		scopes = append(scopes, scope)
-		return queries, scopes
-	}
-
-	// Core -> Socket
-	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
-		sockets, _ := topology.GetSocketsFromCores(hwthreads)
-		for _, socket := range sockets {
-			queries = append(queries, APIQuery{
-				Metric:     metric,
-				Hostname:   hostname,
-				Aggregate:  true,
-				Type:       &coreString,
-				TypeIds:    intToStringSlice(topology.Socket[socket]),
-				Resolution: resolution,
-			})
-			scopes = append(scopes, scope)
-		}
-		return queries, scopes
-	}
-
-	// Core -> Node
-	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
-		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
-		queries = append(queries, APIQuery{
-			Metric:     metric,
-			Hostname:   hostname,
-			Aggregate:  true,
-			Type:       &coreString,
-			TypeIds:    intToStringSlice(cores),
-			Resolution: resolution,
-		})
-		scopes = append(scopes, scope)
-		return queries, scopes
-	}
-
-	// MemoryDomain -> MemoryDomain
-	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
-		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
-		queries = append(queries, APIQuery{
-			Metric:     metric,
-			Hostname:   hostname,
-			Aggregate:  false,
-			Type:       &memoryDomainString,
-			TypeIds:    intToStringSlice(memDomains),
-			Resolution: resolution,
-		})
-		scopes = append(scopes, scope)
-		return queries, scopes
-	}
-
-	// MemoryDomain -> Node
-	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
-		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
-		queries = append(queries, APIQuery{
-			Metric:     metric,
-			Hostname:   hostname,
-			Aggregate:  true,
-			Type:       &memoryDomainString,
-			TypeIds:    intToStringSlice(memDomains),
-			Resolution: resolution,
-		})
-		scopes = append(scopes, scope)
-		return queries, scopes
-	}
-
-	// MemoryDomain -> Socket
-	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket {
-		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
-		socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains)
-		if err != nil {
-			cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err)
-			// Rare Error Case -> Still Continue -> Return Empty Slices
-			return queries, scopes
-		}
-
-		// Create a query for each socket
-		for _, domains := range socketToDomains {
-			queries = append(queries, APIQuery{
-				Metric:     metric,
-				Hostname:   hostname,
-				Aggregate:  true,
-				Type:       &memoryDomainString,
-				TypeIds:    intToStringSlice(domains),
-				Resolution: resolution,
-			})
-			// Add scope for each query, not just once
-			scopes = append(scopes, scope)
-		}
-		return queries, scopes
-	}
-
-	// Socket -> Socket
-	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
-		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
-		queries = append(queries, APIQuery{
-			Metric:     metric,
-			Hostname:   hostname,
-			Aggregate:  false,
-			Type:       &socketString,
-			TypeIds:    intToStringSlice(sockets),
-			Resolution: resolution,
-		})
-		scopes = append(scopes, scope)
-		return queries, scopes
-	}
-
-	// Socket -> Node
-	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
-		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
-		queries = append(queries, APIQuery{
-			Metric:     metric,
-			Hostname:   hostname,
-			Aggregate:  true,
-			Type:       &socketString,
-			TypeIds:    intToStringSlice(sockets),
-			Resolution: resolution,
-		})
-		scopes = append(scopes, scope)
-		return queries, scopes
-	}
-
-	// Node -> Node
-	if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
-		queries = append(queries, APIQuery{
-			Metric:     metric,
-			Hostname:   hostname,
-			Resolution: resolution,
-		})
-		scopes = append(scopes, scope)
-		return queries, scopes
-	}
-
-	// Unhandled Case -> Error -> Return nils
-	return nil, nil
-}
-
-// intToStringSlice converts a slice of integers to a slice of strings.
-// Used to convert hardware IDs (core IDs, socket IDs, etc.) to the string format required by the API.
-func intToStringSlice(is []int) []string {
-	ss := make([]string, len(is))
-	for i, x := range is {
-		ss[i] = strconv.Itoa(x)
-	}
-	return ss
-}

internal/metricstoreclient/cc-metric-store.go

@@ -63,7 +63,7 @@ import (
 	"time"
 
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
-	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
+	ms "github.com/ClusterCockpit/cc-backend/pkg/metricstore"
 	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
 	"github.com/ClusterCockpit/cc-lib/v2/schema"
 )
@@ -331,7 +331,7 @@ func (ccms *CCMetricStore) LoadData(
 				}
 			}
 
-			sanitizeStats(&res.Avg, &res.Min, &res.Max)
+			ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)
 
 			jobMetric.Series = append(jobMetric.Series, schema.Series{
 				Hostname: query.Hostname,
@@ -494,7 +494,7 @@ func (ccms *CCMetricStore) LoadScopedStats(
 				}
 			}
 
-			sanitizeStats(&res.Avg, &res.Min, &res.Max)
+			ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)
 
 			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
 				Hostname: query.Hostname,
@@ -584,7 +584,7 @@ func (ccms *CCMetricStore) LoadNodeData(
 			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
 		}
 
-		sanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)
+		ms.SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)
 
 		hostdata, ok := data[query.Hostname]
 		if !ok {
@@ -756,7 +756,7 @@ func (ccms *CCMetricStore) LoadNodeListData(
 				}
 			}
 
-			sanitizeStats(&res.Avg, &res.Min, &res.Max)
+			ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)
 
 			scopeData.Series = append(scopeData.Series, schema.Series{
 				Hostname: query.Hostname,
@@ -784,8 +784,8 @@ func (ccms *CCMetricStore) LoadNodeListData(
 // returns the per-node health check results.
 func (ccms *CCMetricStore) HealthCheck(cluster string,
 	nodes []string, metrics []string,
-) (map[string]metricstore.HealthCheckResult, error) {
-	req := metricstore.HealthCheckReq{
+) (map[string]ms.HealthCheckResult, error) {
+	req := ms.HealthCheckReq{
 		Cluster:     cluster,
 		Nodes:       nodes,
 		MetricNames: metrics,
@@ -818,7 +818,7 @@ func (ccms *CCMetricStore) HealthCheck(cluster string,
 		return nil, fmt.Errorf("'%s': HTTP Status: %s", endpoint, res.Status)
 	}
 
-	var results map[string]metricstore.HealthCheckResult
+	var results map[string]ms.HealthCheckResult
 	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&results); err != nil {
 		cclog.Errorf("Error while decoding health check response: %s", err.Error())
 		return nil, err
@@ -827,16 +827,6 @@ func (ccms *CCMetricStore) HealthCheck(cluster string,
 	return results, nil
 }
 
-// sanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
-// Regular float64 values cannot be JSONed when NaN.
-func sanitizeStats(avg, min, max *schema.Float) {
-	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
-		*avg = schema.Float(0)
-		*min = schema.Float(0)
-		*max = schema.Float(0)
-	}
-}
-
 // hasNaNStats returns true if any of the statistics contain NaN values.
 func hasNaNStats(avg, min, max schema.Float) bool {
 	return avg.IsNaN() || min.IsNaN() || max.IsNaN()