     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstoreclient provides a client for querying the cc-metric-store time series database.
     7→//
     8→// The cc-metric-store is a high-performance time series database optimized for HPC metric data.
     9→// This client handles HTTP communication, query construction, scope transformations, and data retrieval
    10→// for job and node metrics across different metric scopes (node, socket, core, hwthread, accelerator).
    11→//
    12→// # Architecture
    13→//
    14→// The package is split into two main components:
    15→//   - Client Operations (cc-metric-store.go): HTTP client, request handling, data loading methods
    16→//   - Query Building (cc-metric-store-queries.go): Query construction and scope transformation logic
    17→//
    18→// # Basic Usage
    19→//
    20→//	store := NewCCMetricStore("http://localhost:8080", "jwt-token")
    21→//
    22→//	// Load job data
    23→//	jobData, err := store.LoadData(job, metrics, scopes, ctx, resolution)
    24→//	if err != nil {
    25→//	    log.Fatal(err)
    26→//	}
    27→//
    28→// # Metric Scopes
    29→//
    30→// The client supports hierarchical metric scopes that map to HPC hardware topology:
    31→//   - MetricScopeAccelerator: GPU/accelerator level metrics
    32→//   - MetricScopeHWThread: Hardware thread (SMT) level metrics
    33→//   - MetricScopeCore: CPU core level metrics
    34→//   - MetricScopeSocket: CPU socket level metrics
    35→//   - MetricScopeMemoryDomain: NUMA domain level metrics
    36→//   - MetricScopeNode: Full node level metrics
    37→//
    38→// The client automatically handles scope transformations, aggregating finer-grained metrics
    39→// to coarser scopes when needed (e.g., aggregating core metrics to socket level).
    40→//
    41→// # Error Handling
    42→//
    43→// The client supports partial errors - if some queries fail, it returns both the successful
    44→// data and an error listing the failed queries. This allows processing partial results
    45→// when some nodes or metrics are temporarily unavailable.
    46→//
    47→// # API Versioning
    48→//
    49→// The client uses cc-metric-store API v2, which includes support for:
    50→//   - Data resampling for bandwidth optimization
    51→//   - Multi-scope queries in a single request
    52→//   - Aggregation across hardware topology levels
    53→package metricstoreclient
    54→
    55→import (
    56→	"bufio"
    57→	"bytes"
    58→	"context"
    59→	"encoding/json"
    60→	"fmt"
    61→	"net/http"
    62→	"strings"
    63→	"time"
    64→
    65→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    66→	ms "github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    67→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    68→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    69→)
    70→
    71→// CCMetricStore is the HTTP client for communicating with cc-metric-store.
    72→// It manages connection details, authentication, and provides methods for querying metrics.
    73→type CCMetricStore struct {
    74→	client        http.Client                 // HTTP client with 10-second timeout
    75→	jwt           string                      // JWT Bearer token for authentication
    76→	url           string                      // Base URL of cc-metric-store instance
    77→	queryEndpoint string                      // Full URL to query API endpoint
    78→	topologyCache map[string]*schema.Topology // cluster -> topology cache
    79→}
    80→
    81→// APIQueryRequest represents a request to the cc-metric-store query API.
    82→// It supports both explicit queries and "for-all-nodes" bulk queries.
    83→type APIQueryRequest struct {
    84→	Cluster     string     `json:"cluster"`       // Target cluster name
    85→	Queries     []APIQuery `json:"queries"`       // Explicit list of metric queries
    86→	ForAllNodes []string   `json:"for-all-nodes"` // Metrics to query for all nodes
    87→	From        int64      `json:"from"`          // Start time (Unix timestamp)
    88→	To          int64      `json:"to"`            // End time (Unix timestamp)
    89→	WithStats   bool       `json:"with-stats"`    // Include min/avg/max statistics
    90→	WithData    bool       `json:"with-data"`     // Include time series data points
    91→}
    92→
    93→// APIQuery specifies a single metric query with optional scope filtering.
    94→// Type and TypeIds define the hardware scope (core, socket, accelerator, etc.).
    95→type APIQuery struct {
    96→	Type       *string  `json:"type,omitempty"`        // Scope type (e.g., "core", "socket")
    97→	SubType    *string  `json:"subtype,omitempty"`     // Sub-scope type (reserved for future use)
    98→	Metric     string   `json:"metric"`                // Metric name
    99→	Hostname   string   `json:"host"`                  // Target hostname
   100→	Resolution int      `json:"resolution"`            // Data resolution in seconds (0 = native)
   101→	TypeIds    []string `json:"type-ids,omitempty"`    // IDs for the scope type (e.g., core IDs)
   102→	SubTypeIds []string `json:"subtype-ids,omitempty"` // IDs for sub-scope (reserved)
   103→	Aggregate  bool     `json:"aggreg"`                // Aggregate across TypeIds
   104→}
   105→
   106→// APIQueryResponse contains the results from a cc-metric-store query.
   107→// Results align with the Queries slice by index.
   108→type APIQueryResponse struct {
   109→	Queries []APIQuery        `json:"queries,omitempty"` // Echoed queries (for bulk requests)
   110→	Results [][]APIMetricData `json:"results"`           // Result data, indexed by query
   111→}
   112→
   113→// APIMetricData represents time series data and statistics for a single metric series.
   114→// Error is set if this particular series failed to load.
   115→type APIMetricData struct {
   116→	Error      *string        `json:"error"`      // Error message if query failed
   117→	Data       []schema.Float `json:"data"`       // Time series data points
   118→	From       int64          `json:"from"`       // Actual start time of data
   119→	To         int64          `json:"to"`         // Actual end time of data
   120→	Resolution int            `json:"resolution"` // Actual resolution of data in seconds
   121→	Avg        schema.Float   `json:"avg"`        // Average value across time range
   122→	Min        schema.Float   `json:"min"`        // Minimum value in time range
   123→	Max        schema.Float   `json:"max"`        // Maximum value in time range
   124→}
   125→
   126→// NewCCMetricStore creates and initializes a new (external) CCMetricStore client.
   127→// The url parameter should include the protocol and port (e.g., "http://localhost:8080").
   128→// The token parameter is a JWT used for Bearer authentication; pass empty string if auth is disabled.
   129→func NewCCMetricStore(url string, token string) *CCMetricStore {
   130→	return &CCMetricStore{
   131→		url:           url,
   132→		queryEndpoint: fmt.Sprintf("%s/api/query", url),
   133→		jwt:           token,
   134→		client: http.Client{
   135→			Timeout: 10 * time.Second,
   136→		},
   137→		topologyCache: make(map[string]*schema.Topology),
   138→	}
   139→}
   140→
   141→// doRequest executes an HTTP POST request to the cc-metric-store query API.
   142→// It handles JSON encoding/decoding, authentication, and API versioning.
   143→// The request body is automatically closed to prevent resource leaks.
   144→func (ccms *CCMetricStore) doRequest(
   145→	ctx context.Context,
   146→	body *APIQueryRequest,
   147→) (*APIQueryResponse, error) {
   148→	buf := &bytes.Buffer{}
   149→	if err := json.NewEncoder(buf).Encode(body); err != nil {
   150→		cclog.Errorf("Error while encoding request body: %s", err.Error())
   151→		return nil, err
   152→	}
   153→
   154→	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf)
   155→	if err != nil {
   156→		cclog.Errorf("Error while building request body: %s", err.Error())
   157→		return nil, err
   158→	}
   159→	if ccms.jwt != "" {
   160→		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   161→	}
   162→
   163→	// versioning the cc-metric-store query API.
   164→	// v2 = data with resampling
   165→	// v1 = data without resampling
   166→	q := req.URL.Query()
   167→	q.Add("version", "v2")
   168→	req.URL.RawQuery = q.Encode()
   169→
   170→	res, err := ccms.client.Do(req)
   171→	if err != nil {
   172→		cclog.Errorf("Error while performing request: %s", err.Error())
   173→		return nil, err
   174→	}
   175→	defer res.Body.Close()
   176→
   177→	if res.StatusCode != http.StatusOK {
   178→		return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status)
   179→	}
   180→
   181→	var resBody APIQueryResponse
   182→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
   183→		cclog.Errorf("Error while decoding result body: %s", err.Error())
   184→		return nil, err
   185→	}
   186→
   187→	return &resBody, nil
   188→}
   189→
   190→// getTopology returns the topology for a given cluster and subcluster, caching it if not already present
   191→func (ccms *CCMetricStore) getTopology(cluster, subCluster string) (*schema.Topology, error) {
   192→	cacheKey := fmt.Sprintf("%s:%s", cluster, subCluster)
   193→	if topology, ok := ccms.topologyCache[cacheKey]; ok {
   194→		return topology, nil
   195→	}
   196→
   197→	subcluster, err := archive.GetSubCluster(cluster, subCluster)
   198→	if err != nil {
   199→		return nil, err
   200→	}
   201→
   202→	ccms.topologyCache[cacheKey] = &subcluster.Topology
   203→	return &subcluster.Topology, nil
   204→}
   205→
   206→// getTopologyByNode returns the topology for a given cluster and node, caching it if not already present
   207→func (ccms *CCMetricStore) getTopologyByNode(cluster, node string) (*schema.Topology, error) {
   208→	subCluster, err := archive.GetSubClusterByNode(cluster, node)
   209→	if err != nil {
   210→		return nil, err
   211→	}
   212→
   213→	return ccms.getTopology(cluster, subCluster)
   214→}
   215→
   216→// LoadData retrieves time series data and statistics for the specified job and metrics.
   217→// It queries data for the job's time range and resources, handling scope transformations automatically.
   218→//
   219→// Parameters:
   220→//   - job: Job metadata including cluster, time range, and allocated resources
   221→//   - metrics: List of metric names to retrieve
   222→//   - scopes: Requested metric scopes (node, socket, core, etc.)
   223→//   - ctx: Context for cancellation and timeouts
   224→//   - resolution: Data resolution in seconds (0 for native resolution)
   225→//
   226→// Returns JobData organized as: metric -> scope -> series list.
   227→// Supports partial errors: returns available data even if some queries fail.
   228→func (ccms *CCMetricStore) LoadData(
   229→	job *schema.Job,
   230→	metrics []string,
   231→	scopes []schema.MetricScope,
   232→	ctx context.Context,
   233→	resolution int,
   234→) (schema.JobData, error) {
   235→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
   236→	if err != nil {
   237→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   238→		return nil, err
   239→	}
   240→
   241→	// Verify assignment is correct - log any inconsistencies for debugging
   242→	if len(queries) != len(assignedScope) {
   243→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
   244→			len(queries), len(assignedScope))
   245→	}
   246→
   247→	req := APIQueryRequest{
   248→		Cluster:   job.Cluster,
   249→		From:      job.StartTime,
   250→		To:        job.StartTime + int64(job.Duration),
   251→		Queries:   queries,
   252→		WithStats: true,
   253→		WithData:  true,
   254→	}
   255→
   256→	resBody, err := ccms.doRequest(ctx, &req)
   257→	if err != nil {
   258→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   259→		return nil, err
   260→	}
   261→
   262→	var errors []string
   263→	jobData := make(schema.JobData)
   264→
   265→	// Add safety check for potential index out of range errors
   266→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   267→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   268→			len(req.Queries), len(resBody.Results), len(assignedScope))
   269→		if len(resBody.Results) > len(req.Queries) {
   270→			resBody.Results = resBody.Results[:len(req.Queries)]
   271→		}
   272→		if len(assignedScope) > len(req.Queries) {
   273→			assignedScope = assignedScope[:len(req.Queries)]
   274→		}
   275→	}
   276→
   277→	for i, row := range resBody.Results {
   278→		// Safety check to prevent index out of range errors
   279→		if i >= len(req.Queries) || i >= len(assignedScope) {
   280→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   281→				i, len(req.Queries), len(assignedScope))
   282→			continue
   283→		}
   284→
   285→		query := req.Queries[i]
   286→		metric := query.Metric
   287→		scope := assignedScope[i]
   288→		mc := archive.GetMetricConfig(job.Cluster, metric)
   289→
   290→		if mc == nil {
   291→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   292→			continue
   293→		}
   294→
   295→		if _, ok := jobData[metric]; !ok {
   296→			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
   297→		}
   298→
   299→		res := mc.Timestep
   300→		if len(row) > 0 {
   301→			res = row[0].Resolution
   302→		}
   303→
   304→		jobMetric, ok := jobData[metric][scope]
   305→		if !ok {
   306→			jobMetric = &schema.JobMetric{
   307→				Unit:     mc.Unit,
   308→				Timestep: res,
   309→				Series:   make([]schema.Series, 0),
   310→			}
   311→			jobData[metric][scope] = jobMetric
   312→		}
   313→
   314→		for ndx, res := range row {
   315→			if res.Error != nil {
   316→				/* Build list for "partial errors", if any */
   317→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   318→				continue
   319→			}
   320→
   321→			id := (*string)(nil)
   322→			if query.Type != nil {
   323→				// Check if ndx is within the bounds of TypeIds slice
   324→				if ndx < len(query.TypeIds) {
   325→					id = new(string)
   326→					*id = query.TypeIds[ndx]
   327→				} else {
   328→					// Log the error but continue processing
   329→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   330→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   331→				}
   332→			}
   333→
   334→			ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)
   335→
   336→			jobMetric.Series = append(jobMetric.Series, schema.Series{
   337→				Hostname: query.Hostname,
   338→				ID:       id,
   339→				Statistics: schema.MetricStatistics{
   340→					Avg: float64(res.Avg),
   341→					Min: float64(res.Min),
   342→					Max: float64(res.Max),
   343→				},
   344→				Data: res.Data,
   345→			})
   346→		}
   347→
   348→		// So that one can later check len(jobData):
   349→		if len(jobMetric.Series) == 0 {
   350→			delete(jobData[metric], scope)
   351→			if len(jobData[metric]) == 0 {
   352→				delete(jobData, metric)
   353→			}
   354→		}
   355→	}
   356→
   357→	if len(errors) != 0 {
   358→		/* Returns list for "partial errors" */
   359→		return jobData, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   360→	}
   361→	return jobData, nil
   362→}
   363→
   364→// LoadStats retrieves min/avg/max statistics for job metrics at node scope.
   365→// This is faster than LoadData when only statistical summaries are needed (no time series data).
   366→//
   367→// Returns statistics organized as: metric -> hostname -> statistics.
   368→func (ccms *CCMetricStore) LoadStats(
   369→	job *schema.Job,
   370→	metrics []string,
   371→	ctx context.Context,
   372→) (map[string]map[string]schema.MetricStatistics, error) {
   373→	queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization?
   374→	if err != nil {
   375→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
   376→		return nil, err
   377→	}
   378→
   379→	req := APIQueryRequest{
   380→		Cluster:   job.Cluster,
   381→		From:      job.StartTime,
   382→		To:        job.StartTime + int64(job.Duration),
   383→		Queries:   queries,
   384→		WithStats: true,
   385→		WithData:  false,
   386→	}
   387→
   388→	resBody, err := ccms.doRequest(ctx, &req)
   389→	if err != nil {
   390→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   391→		return nil, err
   392→	}
   393→
   394→	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
   395→	for i, res := range resBody.Results {
   396→		if len(res) == 0 {
   397→			// No Data Found For Metric, Logged in FetchData to Warn
   398→			continue
   399→		}
   400→		query := req.Queries[i]
   401→		metric := query.Metric
   402→		data := res[0]
   403→		if data.Error != nil {
   404→			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
   405→			continue
   406→		}
   407→
   408→		metricdata, ok := stats[metric]
   409→		if !ok {
   410→			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
   411→			stats[metric] = metricdata
   412→		}
   413→
   414→		if hasNaNStats(data.Avg, data.Min, data.Max) {
   415→			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
   416→			continue
   417→		}
   418→
   419→		metricdata[query.Hostname] = schema.MetricStatistics{
   420→			Avg: float64(data.Avg),
   421→			Min: float64(data.Min),
   422→			Max: float64(data.Max),
   423→		}
   424→	}
   425→
   426→	return stats, nil
   427→}
   428→
   429→// LoadScopedStats retrieves statistics for job metrics across multiple scopes.
   430→// Used for the Job-View Statistics Table to display per-scope breakdowns.
   431→//
   432→// Returns statistics organized as: metric -> scope -> list of scoped statistics.
   433→// Each scoped statistic includes hostname, hardware ID (if applicable), and min/avg/max values.
   434→func (ccms *CCMetricStore) LoadScopedStats(
   435→	job *schema.Job,
   436→	metrics []string,
   437→	scopes []schema.MetricScope,
   438→	ctx context.Context,
   439→) (schema.ScopedJobStats, error) {
   440→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
   441→	if err != nil {
   442→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   443→		return nil, err
   444→	}
   445→
   446→	req := APIQueryRequest{
   447→		Cluster:   job.Cluster,
   448→		From:      job.StartTime,
   449→		To:        job.StartTime + int64(job.Duration),
   450→		Queries:   queries,
   451→		WithStats: true,
   452→		WithData:  false,
   453→	}
   454→
   455→	resBody, err := ccms.doRequest(ctx, &req)
   456→	if err != nil {
   457→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   458→		return nil, err
   459→	}
   460→
   461→	var errors []string
   462→	scopedJobStats := make(schema.ScopedJobStats)
   463→
   464→	for i, row := range resBody.Results {
   465→		query := req.Queries[i]
   466→		metric := query.Metric
   467→		scope := assignedScope[i]
   468→
   469→		if _, ok := scopedJobStats[metric]; !ok {
   470→			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
   471→		}
   472→
   473→		if _, ok := scopedJobStats[metric][scope]; !ok {
   474→			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
   475→		}
   476→
   477→		for ndx, res := range row {
   478→			if res.Error != nil {
   479→				/* Build list for "partial errors", if any */
   480→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   481→				continue
   482→			}
   483→
   484→			id := (*string)(nil)
   485→			if query.Type != nil {
   486→				// Check if ndx is within the bounds of TypeIds slice
   487→				if ndx < len(query.TypeIds) {
   488→					id = new(string)
   489→					*id = query.TypeIds[ndx]
   490→				} else {
   491→					// Log the error but continue processing
   492→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   493→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   494→				}
   495→			}
   496→
   497→			ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)
   498→
   499→			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
   500→				Hostname: query.Hostname,
   501→				ID:       id,
   502→				Data: &schema.MetricStatistics{
   503→					Avg: float64(res.Avg),
   504→					Min: float64(res.Min),
   505→					Max: float64(res.Max),
   506→				},
   507→			})
   508→		}
   509→
   510→		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
   511→		if len(scopedJobStats[metric][scope]) == 0 {
   512→			delete(scopedJobStats[metric], scope)
   513→			if len(scopedJobStats[metric]) == 0 {
   514→				delete(scopedJobStats, metric)
   515→			}
   516→		}
   517→	}
   518→
   519→	if len(errors) != 0 {
   520→		/* Returns list for "partial errors" */
   521→		return scopedJobStats, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   522→	}
   523→	return scopedJobStats, nil
   524→}
   525→
   526→// LoadNodeData retrieves current metric data for specified nodes in a cluster.
   527→// Used for the Systems-View Node-Overview to display real-time node status.
   528→//
   529→// If nodes is nil, queries all metrics for all nodes in the cluster (bulk query).
   530→// Returns data organized as: hostname -> metric -> list of JobMetric (with time series and stats).
   531→func (ccms *CCMetricStore) LoadNodeData(
   532→	cluster string,
   533→	metrics, nodes []string,
   534→	scopes []schema.MetricScope,
   535→	from, to time.Time,
   536→	ctx context.Context,
   537→) (map[string]map[string][]*schema.JobMetric, error) {
   538→	req := APIQueryRequest{
   539→		Cluster:   cluster,
   540→		From:      from.Unix(),
   541→		To:        to.Unix(),
   542→		WithStats: true,
   543→		WithData:  true,
   544→	}
   545→
   546→	if nodes == nil {
   547→		req.ForAllNodes = append(req.ForAllNodes, metrics...)
   548→	} else {
   549→		for _, node := range nodes {
   550→			for _, metric := range metrics {
   551→				req.Queries = append(req.Queries, APIQuery{
   552→					Hostname:   node,
   553→					Metric:     metric,
   554→					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
   555→				})
   556→			}
   557→		}
   558→	}
   559→
   560→	resBody, err := ccms.doRequest(ctx, &req)
   561→	if err != nil {
   562→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   563→		return nil, err
   564→	}
   565→
   566→	var errors []string
   567→	data := make(map[string]map[string][]*schema.JobMetric)
   568→	for i, res := range resBody.Results {
   569→		if len(res) == 0 {
   570→			// No Data Found For Metric, Logged in FetchData to Warn
   571→			continue
   572→		}
   573→
   574→		var query APIQuery
   575→		if resBody.Queries != nil {
   576→			query = resBody.Queries[i]
   577→		} else {
   578→			query = req.Queries[i]
   579→		}
   580→
   581→		metric := query.Metric
   582→		qdata := res[0]
   583→		if qdata.Error != nil {
   584→			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
   585→		}
   586→
   587→		ms.SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)
   588→
   589→		hostdata, ok := data[query.Hostname]
   590→		if !ok {
   591→			hostdata = make(map[string][]*schema.JobMetric)
   592→			data[query.Hostname] = hostdata
   593→		}
   594→
   595→		mc := archive.GetMetricConfig(cluster, metric)
   596→		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
   597→			Unit:     mc.Unit,
   598→			Timestep: mc.Timestep,
   599→			Series: []schema.Series{
   600→				{
   601→					Hostname: query.Hostname,
   602→					Data:     qdata.Data,
   603→					Statistics: schema.MetricStatistics{
   604→						Avg: float64(qdata.Avg),
   605→						Min: float64(qdata.Min),
   606→						Max: float64(qdata.Max),
   607→					},
   608→				},
   609→			},
   610→		})
   611→	}
   612→
   613→	if len(errors) != 0 {
   614→		/* Returns list of "partial errors" */
   615→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   616→	}
   617→
   618→	return data, nil
   619→}
   620→
   621→// LoadNodeListData retrieves paginated node metrics for the Systems-View Node-List.
   622→//
   623→// Supports filtering by subcluster and node name pattern. The nodeFilter performs
   624→// substring matching on hostnames.
   625→//
   626→// Returns:
   627→//   - Node data organized as: hostname -> JobData (metric -> scope -> series)
   628→//   - Total node count (before pagination)
   629→//   - HasNextPage flag indicating if more pages are available
   630→//   - Error (may be partial error with some data returned)
   631→func (ccms *CCMetricStore) LoadNodeListData(
   632→	cluster, subCluster string,
   633→	nodes []string,
   634→	metrics []string,
   635→	scopes []schema.MetricScope,
   636→	resolution int,
   637→	from, to time.Time,
   638→	ctx context.Context,
   639→) (map[string]schema.JobData, error) {
   640→	queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
   641→	if err != nil {
   642→		cclog.Errorf("Error while building node queries for Cluster %s, SubCluster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
   643→		return nil, err
   644→	}
   645→
   646→	// Verify assignment is correct - log any inconsistencies for debugging
   647→	if len(queries) != len(assignedScope) {
   648→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
   649→			len(queries), len(assignedScope))
   650→	}
   651→
   652→	req := APIQueryRequest{
   653→		Cluster:   cluster,
   654→		Queries:   queries,
   655→		From:      from.Unix(),
   656→		To:        to.Unix(),
   657→		WithStats: true,
   658→		WithData:  true,
   659→	}
   660→
   661→	resBody, err := ccms.doRequest(ctx, &req)
   662→	if err != nil {
   663→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   664→		return nil, err
   665→	}
   666→
   667→	var errors []string
   668→	data := make(map[string]schema.JobData)
   669→
   670→	// Add safety check for index out of range issues
   671→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   672→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   673→			len(req.Queries), len(resBody.Results), len(assignedScope))
   674→		if len(resBody.Results) > len(req.Queries) {
   675→			resBody.Results = resBody.Results[:len(req.Queries)]
   676→		}
   677→		if len(assignedScope) > len(req.Queries) {
   678→			assignedScope = assignedScope[:len(req.Queries)]
   679→		}
   680→	}
   681→
   682→	for i, row := range resBody.Results {
   683→		// Safety check to prevent index out of range errors
   684→		if i >= len(req.Queries) || i >= len(assignedScope) {
   685→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   686→				i, len(req.Queries), len(assignedScope))
   687→			continue
   688→		}
   689→
   690→		var query APIQuery
   691→		if resBody.Queries != nil {
   692→			if i < len(resBody.Queries) {
   693→				query = resBody.Queries[i]
   694→			} else {
   695→				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
   696→					i, len(resBody.Queries))
   697→				continue
   698→			}
   699→		} else {
   700→			query = req.Queries[i]
   701→		}
   702→
   703→		metric := query.Metric
   704→		scope := assignedScope[i]
   705→		mc := archive.GetMetricConfig(cluster, metric)
   706→		if mc == nil {
   707→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   708→			continue
   709→		}
   710→
   711→		res := mc.Timestep
   712→		if len(row) > 0 {
   713→			res = row[0].Resolution
   714→		}
   715→
   716→		// Init Nested Map Data Structures If Not Found
   717→		hostData, ok := data[query.Hostname]
   718→		if !ok {
   719→			hostData = make(schema.JobData)
   720→			data[query.Hostname] = hostData
   721→		}
   722→
   723→		metricData, ok := hostData[metric]
   724→		if !ok {
   725→			metricData = make(map[schema.MetricScope]*schema.JobMetric)
   726→			data[query.Hostname][metric] = metricData
   727→		}
   728→
   729→		scopeData, ok := metricData[scope]
   730→		if !ok {
   731→			scopeData = &schema.JobMetric{
   732→				Unit:     mc.Unit,
   733→				Timestep: res,
   734→				Series:   make([]schema.Series, 0),
   735→			}
   736→			data[query.Hostname][metric][scope] = scopeData
   737→		}
   738→
   739→		for ndx, res := range row {
   740→			if res.Error != nil {
   741→				/* Build list for "partial errors", if any */
   742→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   743→				continue
   744→			}
   745→
   746→			id := (*string)(nil)
   747→			if query.Type != nil {
   748→				// Check if ndx is within the bounds of TypeIds slice
   749→				if ndx < len(query.TypeIds) {
   750→					id = new(string)
   751→					*id = query.TypeIds[ndx]
   752→				} else {
   753→					// Log the error but continue processing
   754→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   755→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   756→				}
   757→			}
   758→
   759→			ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)
   760→
   761→			scopeData.Series = append(scopeData.Series, schema.Series{
   762→				Hostname: query.Hostname,
   763→				ID:       id,
   764→				Statistics: schema.MetricStatistics{
   765→					Avg: float64(res.Avg),
   766→					Min: float64(res.Min),
   767→					Max: float64(res.Max),
   768→				},
   769→				Data: res.Data,
   770→			})
   771→		}
   772→	}
   773→
   774→	if len(errors) != 0 {
   775→		/* Returns list of "partial errors" */
   776→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   777→	}
   778→
   779→	return data, nil
   780→}
   781→
   782→// HealthCheck queries the external cc-metric-store's health check endpoint.
   783→// It sends a HealthCheckReq as the request body to /api/healthcheck and
   784→// returns the per-node health check results.
   785→func (ccms *CCMetricStore) HealthCheck(cluster string,
   786→	nodes []string, metrics []string,
   787→) (map[string]ms.HealthCheckResult, error) {
   788→	req := ms.HealthCheckReq{
   789→		Cluster:     cluster,
   790→		Nodes:       nodes,
   791→		MetricNames: metrics,
   792→	}
   793→
   794→	buf := &bytes.Buffer{}
   795→	if err := json.NewEncoder(buf).Encode(req); err != nil {
   796→		cclog.Errorf("Error while encoding health check request body: %s", err.Error())
   797→		return nil, err
   798→	}
   799→
   800→	endpoint := fmt.Sprintf("%s/api/healthcheck", ccms.url)
   801→	httpReq, err := http.NewRequest(http.MethodGet, endpoint, buf)
   802→	if err != nil {
   803→		cclog.Errorf("Error while building health check request: %s", err.Error())
   804→		return nil, err
   805→	}
   806→	if ccms.jwt != "" {
   807→		httpReq.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   808→	}
   809→
   810→	res, err := ccms.client.Do(httpReq)
   811→	if err != nil {
   812→		cclog.Errorf("Error while performing health check request: %s", err.Error())
   813→		return nil, err
   814→	}
   815→	defer res.Body.Close()
   816→
   817→	if res.StatusCode != http.StatusOK {
   818→		return nil, fmt.Errorf("'%s': HTTP Status: %s", endpoint, res.Status)
   819→	}
   820→
   821→	var results map[string]ms.HealthCheckResult
   822→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&results); err != nil {
   823→		cclog.Errorf("Error while decoding health check response: %s", err.Error())
   824→		return nil, err
   825→	}
   826→
   827→	return results, nil
   828→}
   829→
   830→// hasNaNStats returns true if any of the statistics contain NaN values.
   831→func hasNaNStats(avg, min, max schema.Float) bool {
   832→	return avg.IsNaN() || min.IsNaN() || max.IsNaN()
   833→}
   834→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstoreclient provides a client for querying the cc-metric-store time series database.
//
// The cc-metric-store is a high-performance time series database optimized for HPC metric data.
// This client handles HTTP communication, query construction, scope transformations, and data retrieval
// for job and node metrics across different metric scopes (node, socket, core, hwthread, accelerator).
//
// # Architecture
//
// The package is split into two main components:
//   - Client Operations (cc-metric-store.go): HTTP client, request handling, data loading methods
//   - Query Building (cc-metric-store-queries.go): Query construction and scope transformation logic
//
// # Basic Usage
//
//	store := NewCCMetricStore("http://localhost:8080", "jwt-token")
//
//	// Load job data
//	jobData, err := store.LoadData(job, metrics, scopes, ctx, resolution)
//	if err != nil {
//	    log.Fatal(err)
//	}
//
// # Metric Scopes
//
// The client supports hierarchical metric scopes that map to HPC hardware topology:
//   - MetricScopeAccelerator: GPU/accelerator level metrics
//   - MetricScopeHWThread: Hardware thread (SMT) level metrics
//   - MetricScopeCore: CPU core level metrics
//   - MetricScopeSocket: CPU socket level metrics
//   - MetricScopeMemoryDomain: NUMA domain level metrics
//   - MetricScopeNode: Full node level metrics
//
// The client automatically handles scope transformations, aggregating finer-grained metrics
// to coarser scopes when needed (e.g., aggregating core metrics to socket level).
//
// # Error Handling
//
// The client supports partial errors - if some queries fail, it returns both the successful
// data and an error listing the failed queries. This allows processing partial results
// when some nodes or metrics are temporarily unavailable.
//
// # API Versioning
//
// The client uses cc-metric-store API v2, which includes support for:
//   - Data resampling for bandwidth optimization
//   - Multi-scope queries in a single request
//   - Aggregation across hardware topology levels
package metricstoreclient

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	ms "github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// CCMetricStore is the HTTP client for communicating with cc-metric-store.
// It manages connection details, authentication, and provides methods for querying metrics.
type CCMetricStore struct {
	client        http.Client                 // HTTP client with 10-second timeout
	jwt           string                      // JWT Bearer token for authentication
	url           string                      // Base URL of cc-metric-store instance
	queryEndpoint string                      // Full URL to query API endpoint
	topologyCache map[string]*schema.Topology // cluster -> topology cache
}

// APIQueryRequest represents a request to the cc-metric-store query API.
// It supports both explicit queries and "for-all-nodes" bulk queries.
type APIQueryRequest struct {
	Cluster     string     `json:"cluster"`       // Target cluster name
	Queries     []APIQuery `json:"queries"`       // Explicit list of metric queries
	ForAllNodes []string   `json:"for-all-nodes"` // Metrics to query for all nodes
	From        int64      `json:"from"`          // Start time (Unix timestamp)
	To          int64      `json:"to"`            // End time (Unix timestamp)
	WithStats   bool       `json:"with-stats"`    // Include min/avg/max statistics
	WithData    bool       `json:"with-data"`     // Include time series data points
}

// APIQuery specifies a single metric query with optional scope filtering.
// Type and TypeIds define the hardware scope (core, socket, accelerator, etc.).
type APIQuery struct {
	Type       *string  `json:"type,omitempty"`        // Scope type (e.g., "core", "socket")
	SubType    *string  `json:"subtype,omitempty"`     // Sub-scope type (reserved for future use)
	Metric     string   `json:"metric"`                // Metric name
	Hostname   string   `json:"host"`                  // Target hostname
	Resolution int      `json:"resolution"`            // Data resolution in seconds (0 = native)
	TypeIds    []string `json:"type-ids,omitempty"`    // IDs for the scope type (e.g., core IDs)
	SubTypeIds []string `json:"subtype-ids,omitempty"` // IDs for sub-scope (reserved)
	Aggregate  bool     `json:"aggreg"`                // Aggregate across TypeIds
}

// APIQueryResponse contains the results from a cc-metric-store query.
// Results align with the Queries slice by index.
type APIQueryResponse struct {
	Queries []APIQuery        `json:"queries,omitempty"` // Echoed queries (for bulk requests)
	Results [][]APIMetricData `json:"results"`           // Result data, indexed by query
}

// APIMetricData represents time series data and statistics for a single metric series.
// Error is set if this particular series failed to load.
type APIMetricData struct {
	Error      *string        `json:"error"`      // Error message if query failed
	Data       []schema.Float `json:"data"`       // Time series data points
	From       int64          `json:"from"`       // Actual start time of data
	To         int64          `json:"to"`         // Actual end time of data
	Resolution int            `json:"resolution"` // Actual resolution of data in seconds
	Avg        schema.Float   `json:"avg"`        // Average value across time range
	Min        schema.Float   `json:"min"`        // Minimum value in time range
	Max        schema.Float   `json:"max"`        // Maximum value in time range
}

// NewCCMetricStore creates and initializes a new (external) CCMetricStore client.
// The url parameter should include the protocol and port (e.g., "http://localhost:8080").
// The token parameter is a JWT used for Bearer authentication; pass empty string if auth is disabled.
func NewCCMetricStore(url string, token string) *CCMetricStore {
	return &CCMetricStore{
		url:           url,
		queryEndpoint: fmt.Sprintf("%s/api/query", url),
		jwt:           token,
		client: http.Client{
			Timeout: 10 * time.Second,
		},
		topologyCache: make(map[string]*schema.Topology),
	}
}

// doRequest executes an HTTP POST request to the cc-metric-store query API.
// It handles JSON encoding/decoding, authentication, and API versioning.
// The request body is automatically closed to prevent resource leaks.
func (ccms *CCMetricStore) doRequest(
	ctx context.Context,
	body *APIQueryRequest,
) (*APIQueryResponse, error) {
	buf := &bytes.Buffer{}
	if err := json.NewEncoder(buf).Encode(body); err != nil {
		cclog.Errorf("Error while encoding request body: %s", err.Error())
		return nil, err
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf)
	if err != nil {
		cclog.Errorf("Error while building request body: %s", err.Error())
		return nil, err
	}
	if ccms.jwt != "" {
		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
	}

	// versioning the cc-metric-store query API.
	// v2 = data with resampling
	// v1 = data without resampling
	q := req.URL.Query()
	q.Add("version", "v2")
	req.URL.RawQuery = q.Encode()

	res, err := ccms.client.Do(req)
	if err != nil {
		cclog.Errorf("Error while performing request: %s", err.Error())
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status)
	}

	var resBody APIQueryResponse
	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
		cclog.Errorf("Error while decoding result body: %s", err.Error())
		return nil, err
	}

	return &resBody, nil
}

// getTopology returns the topology for a given cluster and subcluster, caching it if not already present
func (ccms *CCMetricStore) getTopology(cluster, subCluster string) (*schema.Topology, error) {
	cacheKey := fmt.Sprintf("%s:%s", cluster, subCluster)
	if topology, ok := ccms.topologyCache[cacheKey]; ok {
		return topology, nil
	}

	subcluster, err := archive.GetSubCluster(cluster, subCluster)
	if err != nil {
		return nil, err
	}

	ccms.topologyCache[cacheKey] = &subcluster.Topology
	return &subcluster.Topology, nil
}

// getTopologyByNode returns the topology for a given cluster and node, caching it if not already present
func (ccms *CCMetricStore) getTopologyByNode(cluster, node string) (*schema.Topology, error) {
	subCluster, err := archive.GetSubClusterByNode(cluster, node)
	if err != nil {
		return nil, err
	}

	return ccms.getTopology(cluster, subCluster)
}

// LoadData retrieves time series data and statistics for the specified job and metrics.
// It queries data for the job's time range and resources, handling scope transformations automatically.
//
// Parameters:
//   - job: Job metadata including cluster, time range, and allocated resources
//   - metrics: List of metric names to retrieve
//   - scopes: Requested metric scopes (node, socket, core, etc.)
//   - ctx: Context for cancellation and timeouts
//   - resolution: Data resolution in seconds (0 for native resolution)
//
// Returns JobData organized as: metric -> scope -> series list.
// Supports partial errors: returns available data even if some queries fail.
func (ccms *CCMetricStore) LoadData(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
	resolution int,
) (schema.JobData, error) {
	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  true,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	var errors []string
	jobData := make(schema.JobData)

	// Add safety check for potential index out of range errors
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		// Safety check to prevent index out of range errors
		if i >= len(req.Queries) || i >= len(assignedScope) {
			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
				i, len(req.Queries), len(assignedScope))
			continue
		}

		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(job.Cluster, metric)

		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
			continue
		}

		if _, ok := jobData[metric]; !ok {
			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = row[0].Resolution
		}

		jobMetric, ok := jobData[metric][scope]
		if !ok {
			jobMetric = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			jobData[metric][scope] = jobMetric
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)

			jobMetric.Series = append(jobMetric.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}

		// So that one can later check len(jobData):
		if len(jobMetric.Series) == 0 {
			delete(jobData[metric], scope)
			if len(jobData[metric]) == 0 {
				delete(jobData, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return jobData, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return jobData, nil
}

// LoadStats retrieves min/avg/max statistics for job metrics at node scope.
// This is faster than LoadData when only statistical summaries are needed (no time series data).
//
// Returns statistics organized as: metric -> hostname -> statistics.
func (ccms *CCMetricStore) LoadStats(
	job *schema.Job,
	metrics []string,
	ctx context.Context,
) (map[string]map[string]schema.MetricStatistics, error) {
	queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization?
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
	for i, res := range resBody.Results {
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}
		query := req.Queries[i]
		metric := query.Metric
		data := res[0]
		if data.Error != nil {
			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
			continue
		}

		metricdata, ok := stats[metric]
		if !ok {
			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
			stats[metric] = metricdata
		}

		if hasNaNStats(data.Avg, data.Min, data.Max) {
			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
			continue
		}

		metricdata[query.Hostname] = schema.MetricStatistics{
			Avg: float64(data.Avg),
			Min: float64(data.Min),
			Max: float64(data.Max),
		}
	}

	return stats, nil
}

// LoadScopedStats retrieves statistics for job metrics across multiple scopes.
// Used for the Job-View Statistics Table to display per-scope breakdowns.
//
// Returns statistics organized as: metric -> scope -> list of scoped statistics.
// Each scoped statistic includes hostname, hardware ID (if applicable), and min/avg/max values.
func (ccms *CCMetricStore) LoadScopedStats(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
) (schema.ScopedJobStats, error) {
	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	var errors []string
	scopedJobStats := make(schema.ScopedJobStats)

	for i, row := range resBody.Results {
		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]

		if _, ok := scopedJobStats[metric]; !ok {
			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
		}

		if _, ok := scopedJobStats[metric][scope]; !ok {
			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
				Hostname: query.Hostname,
				ID:       id,
				Data: &schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
			})
		}

		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
		if len(scopedJobStats[metric][scope]) == 0 {
			delete(scopedJobStats[metric], scope)
			if len(scopedJobStats[metric]) == 0 {
				delete(scopedJobStats, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return scopedJobStats, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return scopedJobStats, nil
}

// LoadNodeData retrieves current metric data for specified nodes in a cluster.
// Used for the Systems-View Node-Overview to display real-time node status.
//
// If nodes is nil, queries all metrics for all nodes in the cluster (bulk query).
// Returns data organized as: hostname -> metric -> list of JobMetric (with time series and stats).
func (ccms *CCMetricStore) LoadNodeData(
	cluster string,
	metrics, nodes []string,
	scopes []schema.MetricScope,
	from, to time.Time,
	ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
	req := APIQueryRequest{
		Cluster:   cluster,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	if nodes == nil {
		req.ForAllNodes = append(req.ForAllNodes, metrics...)
	} else {
		for _, node := range nodes {
			for _, metric := range metrics {
				req.Queries = append(req.Queries, APIQuery{
					Hostname:   node,
					Metric:     metric,
					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
				})
			}
		}
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]map[string][]*schema.JobMetric)
	for i, res := range resBody.Results {
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}

		var query APIQuery
		if resBody.Queries != nil {
			query = resBody.Queries[i]
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		qdata := res[0]
		if qdata.Error != nil {
			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
		}

		ms.SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)

		hostdata, ok := data[query.Hostname]
		if !ok {
			hostdata = make(map[string][]*schema.JobMetric)
			data[query.Hostname] = hostdata
		}

		mc := archive.GetMetricConfig(cluster, metric)
		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
			Unit:     mc.Unit,
			Timestep: mc.Timestep,
			Series: []schema.Series{
				{
					Hostname: query.Hostname,
					Data:     qdata.Data,
					Statistics: schema.MetricStatistics{
						Avg: float64(qdata.Avg),
						Min: float64(qdata.Min),
						Max: float64(qdata.Max),
					},
				},
			},
		})
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// LoadNodeListData retrieves paginated node metrics for the Systems-View Node-List.
//
// Supports filtering by subcluster and node name pattern. The nodeFilter performs
// substring matching on hostnames.
//
// Returns:
//   - Node data organized as: hostname -> JobData (metric -> scope -> series)
//   - Total node count (before pagination)
//   - HasNextPage flag indicating if more pages are available
//   - Error (may be partial error with some data returned)
func (ccms *CCMetricStore) LoadNodeListData(
	cluster, subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	from, to time.Time,
	ctx context.Context,
) (map[string]schema.JobData, error) {
	queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
	if err != nil {
		cclog.Errorf("Error while building node queries for Cluster %s, SubCluster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:   cluster,
		Queries:   queries,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]schema.JobData)

	// Add safety check for index out of range issues
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		// Safety check to prevent index out of range errors
		if i >= len(req.Queries) || i >= len(assignedScope) {
			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
				i, len(req.Queries), len(assignedScope))
			continue
		}

		var query APIQuery
		if resBody.Queries != nil {
			if i < len(resBody.Queries) {
				query = resBody.Queries[i]
			} else {
				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
					i, len(resBody.Queries))
				continue
			}
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
			continue
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = row[0].Resolution
		}

		// Init Nested Map Data Structures If Not Found
		hostData, ok := data[query.Hostname]
		if !ok {
			hostData = make(schema.JobData)
			data[query.Hostname] = hostData
		}

		metricData, ok := hostData[metric]
		if !ok {
			metricData = make(map[schema.MetricScope]*schema.JobMetric)
			data[query.Hostname][metric] = metricData
		}

		scopeData, ok := metricData[scope]
		if !ok {
			scopeData = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			data[query.Hostname][metric][scope] = scopeData
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopeData.Series = append(scopeData.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// HealthCheck queries the external cc-metric-store's health check endpoint.
// It sends a HealthCheckReq as the request body to /api/healthcheck and
// returns the per-node health check results.
func (ccms *CCMetricStore) HealthCheck(cluster string,
	nodes []string, metrics []string,
) (map[string]ms.HealthCheckResult, error) {
	req := ms.HealthCheckReq{
		Cluster:     cluster,
		Nodes:       nodes,
		MetricNames: metrics,
	}

	buf := &bytes.Buffer{}
	if err := json.NewEncoder(buf).Encode(req); err != nil {
		cclog.Errorf("Error while encoding health check request body: %s", err.Error())
		return nil, err
	}

	endpoint := fmt.Sprintf("%s/api/healthcheck", ccms.url)
	httpReq, err := http.NewRequest(http.MethodGet, endpoint, buf)
	if err != nil {
		cclog.Errorf("Error while building health check request: %s", err.Error())
		return nil, err
	}
	if ccms.jwt != "" {
		httpReq.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
	}

	res, err := ccms.client.Do(httpReq)
	if err != nil {
		cclog.Errorf("Error while performing health check request: %s", err.Error())
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("'%s': HTTP Status: %s", endpoint, res.Status)
	}

	var results map[string]ms.HealthCheckResult
	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&results); err != nil {
		cclog.Errorf("Error while decoding health check response: %s", err.Error())
		return nil, err
	}

	return results, nil
}

// hasNaNStats returns true if any of the statistics contain NaN values.
func hasNaNStats(avg, min, max schema.Float) bool {
	return avg.IsNaN() || min.IsNaN() || max.IsNaN()
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file contains shared scope transformation logic used by both the internal
     7→// metric store (pkg/metricstore) and the external cc-metric-store client
     8→// (internal/metricstoreclient). It extracts the common algorithm for mapping
     9→// between native metric scopes and requested scopes based on cluster topology.
    10→package metricstore
    11→
    12→import (
    13→	"strconv"
    14→
    15→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    16→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    17→)
    18→
    19→// Pre-converted scope strings avoid repeated string(MetricScope) allocations
    20→// during query construction. Used in ScopeQueryResult.Type field.
    21→var (
    22→	HWThreadString     = string(schema.MetricScopeHWThread)
    23→	CoreString         = string(schema.MetricScopeCore)
    24→	MemoryDomainString = string(schema.MetricScopeMemoryDomain)
    25→	SocketString       = string(schema.MetricScopeSocket)
    26→	AcceleratorString  = string(schema.MetricScopeAccelerator)
    27→)
    28→
    29→// ScopeQueryResult is a package-independent intermediate type returned by
    30→// BuildScopeQueries. Each consumer converts it to their own APIQuery type
    31→// (adding Resolution and any other package-specific fields).
    32→type ScopeQueryResult struct {
    33→	Type      *string
    34→	Metric    string
    35→	Hostname  string
    36→	TypeIds   []string
    37→	Scope     schema.MetricScope
    38→	Aggregate bool
    39→}
    40→
    41→// BuildScopeQueries generates scope query results for a given scope transformation.
    42→// It returns a slice of results and a boolean indicating success.
    43→// An empty slice means an expected exception (skip this combination).
    44→// ok=false means an unhandled case (caller should return an error).
    45→func BuildScopeQueries(
    46→	nativeScope, requestedScope schema.MetricScope,
    47→	metric, hostname string,
    48→	topology *schema.Topology,
    49→	hwthreads []int,
    50→	accelerators []string,
    51→) ([]ScopeQueryResult, bool) {
    52→	scope := nativeScope.Max(requestedScope)
    53→	results := []ScopeQueryResult{}
    54→
    55→	hwthreadsStr := IntToStringSlice(hwthreads)
    56→
    57→	// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
    58→	if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
    59→		if scope != schema.MetricScopeAccelerator {
    60→			// Expected Exception -> Return Empty Slice
    61→			return results, true
    62→		}
    63→
    64→		results = append(results, ScopeQueryResult{
    65→			Metric:    metric,
    66→			Hostname:  hostname,
    67→			Aggregate: false,
    68→			Type:      &AcceleratorString,
    69→			TypeIds:   accelerators,
    70→			Scope:     schema.MetricScopeAccelerator,
    71→		})
    72→		return results, true
    73→	}
    74→
    75→	// Accelerator -> Node
    76→	if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
    77→		if len(accelerators) == 0 {
    78→			// Expected Exception -> Return Empty Slice
    79→			return results, true
    80→		}
    81→
    82→		results = append(results, ScopeQueryResult{
    83→			Metric:    metric,
    84→			Hostname:  hostname,
    85→			Aggregate: true,
    86→			Type:      &AcceleratorString,
    87→			TypeIds:   accelerators,
    88→			Scope:     scope,
    89→		})
    90→		return results, true
    91→	}
    92→
    93→	// HWThread -> HWThread
    94→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
    95→		results = append(results, ScopeQueryResult{
    96→			Metric:    metric,
    97→			Hostname:  hostname,
    98→			Aggregate: false,
    99→			Type:      &HWThreadString,
   100→			TypeIds:   hwthreadsStr,
   101→			Scope:     scope,
   102→		})
   103→		return results, true
   104→	}
   105→
   106→	// HWThread -> Core
   107→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
   108→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   109→		for _, core := range cores {
   110→			results = append(results, ScopeQueryResult{
   111→				Metric:    metric,
   112→				Hostname:  hostname,
   113→				Aggregate: true,
   114→				Type:      &HWThreadString,
   115→				TypeIds:   IntToStringSlice(topology.Core[core]),
   116→				Scope:     scope,
   117→			})
   118→		}
   119→		return results, true
   120→	}
   121→
   122→	// HWThread -> Socket
   123→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
   124→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   125→		for _, socket := range sockets {
   126→			results = append(results, ScopeQueryResult{
   127→				Metric:    metric,
   128→				Hostname:  hostname,
   129→				Aggregate: true,
   130→				Type:      &HWThreadString,
   131→				TypeIds:   IntToStringSlice(topology.Socket[socket]),
   132→				Scope:     scope,
   133→			})
   134→		}
   135→		return results, true
   136→	}
   137→
   138→	// HWThread -> Node
   139→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
   140→		results = append(results, ScopeQueryResult{
   141→			Metric:    metric,
   142→			Hostname:  hostname,
   143→			Aggregate: true,
   144→			Type:      &HWThreadString,
   145→			TypeIds:   hwthreadsStr,
   146→			Scope:     scope,
   147→		})
   148→		return results, true
   149→	}
   150→
   151→	// Core -> Core
   152→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
   153→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   154→		results = append(results, ScopeQueryResult{
   155→			Metric:    metric,
   156→			Hostname:  hostname,
   157→			Aggregate: false,
   158→			Type:      &CoreString,
   159→			TypeIds:   IntToStringSlice(cores),
   160→			Scope:     scope,
   161→		})
   162→		return results, true
   163→	}
   164→
   165→	// Core -> Socket
   166→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
   167→		sockets, _ := topology.GetSocketsFromCores(hwthreads)
   168→		for _, socket := range sockets {
   169→			results = append(results, ScopeQueryResult{
   170→				Metric:    metric,
   171→				Hostname:  hostname,
   172→				Aggregate: true,
   173→				Type:      &CoreString,
   174→				TypeIds:   IntToStringSlice(topology.Socket[socket]),
   175→				Scope:     scope,
   176→			})
   177→		}
   178→		return results, true
   179→	}
   180→
   181→	// Core -> Node
   182→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
   183→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   184→		results = append(results, ScopeQueryResult{
   185→			Metric:    metric,
   186→			Hostname:  hostname,
   187→			Aggregate: true,
   188→			Type:      &CoreString,
   189→			TypeIds:   IntToStringSlice(cores),
   190→			Scope:     scope,
   191→		})
   192→		return results, true
   193→	}
   194→
   195→	// MemoryDomain -> MemoryDomain
   196→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
   197→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   198→		results = append(results, ScopeQueryResult{
   199→			Metric:    metric,
   200→			Hostname:  hostname,
   201→			Aggregate: false,
   202→			Type:      &MemoryDomainString,
   203→			TypeIds:   IntToStringSlice(memDomains),
   204→			Scope:     scope,
   205→		})
   206→		return results, true
   207→	}
   208→
   209→	// MemoryDomain -> Socket
   210→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket {
   211→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   212→		socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains)
   213→		if err != nil {
   214→			cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err)
   215→			// Rare Error Case -> Still Continue -> Return Empty Slice
   216→			return results, true
   217→		}
   218→
   219→		// Create a query for each socket
   220→		for _, domains := range socketToDomains {
   221→			results = append(results, ScopeQueryResult{
   222→				Metric:    metric,
   223→				Hostname:  hostname,
   224→				Aggregate: true,
   225→				Type:      &MemoryDomainString,
   226→				TypeIds:   IntToStringSlice(domains),
   227→				Scope:     scope,
   228→			})
   229→		}
   230→		return results, true
   231→	}
   232→
   233→	// MemoryDomain -> Node
   234→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
   235→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   236→		results = append(results, ScopeQueryResult{
   237→			Metric:    metric,
   238→			Hostname:  hostname,
   239→			Aggregate: true,
   240→			Type:      &MemoryDomainString,
   241→			TypeIds:   IntToStringSlice(memDomains),
   242→			Scope:     scope,
   243→		})
   244→		return results, true
   245→	}
   246→
   247→	// Socket -> Socket
   248→	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
   249→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   250→		results = append(results, ScopeQueryResult{
   251→			Metric:    metric,
   252→			Hostname:  hostname,
   253→			Aggregate: false,
   254→			Type:      &SocketString,
   255→			TypeIds:   IntToStringSlice(sockets),
   256→			Scope:     scope,
   257→		})
   258→		return results, true
   259→	}
   260→
   261→	// Socket -> Node
   262→	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
   263→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   264→		results = append(results, ScopeQueryResult{
   265→			Metric:    metric,
   266→			Hostname:  hostname,
   267→			Aggregate: true,
   268→			Type:      &SocketString,
   269→			TypeIds:   IntToStringSlice(sockets),
   270→			Scope:     scope,
   271→		})
   272→		return results, true
   273→	}
   274→
   275→	// Node -> Node
   276→	if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
   277→		results = append(results, ScopeQueryResult{
   278→			Metric:   metric,
   279→			Hostname: hostname,
   280→			Scope:    scope,
   281→		})
   282→		return results, true
   283→	}
   284→
   285→	// Unhandled Case
   286→	return nil, false
   287→}
   288→
   289→// IntToStringSlice converts a slice of integers to a slice of strings.
   290→// Used to convert hardware thread/core/socket IDs from topology (int) to query TypeIds (string).
   291→// Optimized to reuse a byte buffer for string conversion, reducing allocations.
   292→func IntToStringSlice(is []int) []string {
   293→	if len(is) == 0 {
   294→		return nil
   295→	}
   296→
   297→	ss := make([]string, len(is))
   298→	buf := make([]byte, 0, 16) // Reusable buffer for integer conversion
   299→	for i, x := range is {
   300→		buf = strconv.AppendInt(buf[:0], int64(x), 10)
   301→		ss[i] = string(buf)
   302→	}
   303→	return ss
   304→}
   305→
   306→// SanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
   307→// If ANY of avg/min/max is NaN, ALL three are zeroed for consistency.
   308→func SanitizeStats(avg, min, max *schema.Float) {
   309→	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
   310→		*avg = schema.Float(0)
   311→		*min = schema.Float(0)
   312→		*max = schema.Float(0)
   313→	}
   314→}
   315→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file contains shared scope transformation logic used by both the internal
// metric store (pkg/metricstore) and the external cc-metric-store client
// (internal/metricstoreclient). It extracts the common algorithm for mapping
// between native metric scopes and requested scopes based on cluster topology.
package metricstore

import (
	"strconv"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// Pre-converted scope strings avoid repeated string(MetricScope) allocations
// during query construction. Used in ScopeQueryResult.Type field.
var (
	HWThreadString     = string(schema.MetricScopeHWThread)
	CoreString         = string(schema.MetricScopeCore)
	MemoryDomainString = string(schema.MetricScopeMemoryDomain)
	SocketString       = string(schema.MetricScopeSocket)
	AcceleratorString  = string(schema.MetricScopeAccelerator)
)

// ScopeQueryResult is a package-independent intermediate type returned by
// BuildScopeQueries. Each consumer converts it to their own APIQuery type
// (adding Resolution and any other package-specific fields).
type ScopeQueryResult struct {
	Type      *string
	Metric    string
	Hostname  string
	TypeIds   []string
	Scope     schema.MetricScope
	Aggregate bool
}

// BuildScopeQueries generates scope query results for a given scope transformation.
// It returns a slice of results and a boolean indicating success.
// An empty slice means an expected exception (skip this combination).
// ok=false means an unhandled case (caller should return an error).
func BuildScopeQueries(
	nativeScope, requestedScope schema.MetricScope,
	metric, hostname string,
	topology *schema.Topology,
	hwthreads []int,
	accelerators []string,
) ([]ScopeQueryResult, bool) {
	scope := nativeScope.Max(requestedScope)
	results := []ScopeQueryResult{}

	hwthreadsStr := IntToStringSlice(hwthreads)

	// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
	if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
		if scope != schema.MetricScopeAccelerator {
			// Expected Exception -> Return Empty Slice
			return results, true
		}

		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &AcceleratorString,
			TypeIds:   accelerators,
			Scope:     schema.MetricScopeAccelerator,
		})
		return results, true
	}

	// Accelerator -> Node
	if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
		if len(accelerators) == 0 {
			// Expected Exception -> Return Empty Slice
			return results, true
		}

		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &AcceleratorString,
			TypeIds:   accelerators,
			Scope:     scope,
		})
		return results, true
	}

	// HWThread -> HWThread
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &HWThreadString,
			TypeIds:   hwthreadsStr,
			Scope:     scope,
		})
		return results, true
	}

	// HWThread -> Core
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		for _, core := range cores {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &HWThreadString,
				TypeIds:   IntToStringSlice(topology.Core[core]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// HWThread -> Socket
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		for _, socket := range sockets {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &HWThreadString,
				TypeIds:   IntToStringSlice(topology.Socket[socket]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// HWThread -> Node
	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &HWThreadString,
			TypeIds:   hwthreadsStr,
			Scope:     scope,
		})
		return results, true
	}

	// Core -> Core
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &CoreString,
			TypeIds:   IntToStringSlice(cores),
			Scope:     scope,
		})
		return results, true
	}

	// Core -> Socket
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromCores(hwthreads)
		for _, socket := range sockets {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &CoreString,
				TypeIds:   IntToStringSlice(topology.Socket[socket]),
				Scope:     scope,
			})
		}
		return results, true
	}

	// Core -> Node
	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &CoreString,
			TypeIds:   IntToStringSlice(cores),
			Scope:     scope,
		})
		return results, true
	}

	// MemoryDomain -> MemoryDomain
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &MemoryDomainString,
			TypeIds:   IntToStringSlice(memDomains),
			Scope:     scope,
		})
		return results, true
	}

	// MemoryDomain -> Socket
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains)
		if err != nil {
			cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err)
			// Rare Error Case -> Still Continue -> Return Empty Slice
			return results, true
		}

		// Create a query for each socket
		for _, domains := range socketToDomains {
			results = append(results, ScopeQueryResult{
				Metric:    metric,
				Hostname:  hostname,
				Aggregate: true,
				Type:      &MemoryDomainString,
				TypeIds:   IntToStringSlice(domains),
				Scope:     scope,
			})
		}
		return results, true
	}

	// MemoryDomain -> Node
	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &MemoryDomainString,
			TypeIds:   IntToStringSlice(memDomains),
			Scope:     scope,
		})
		return results, true
	}

	// Socket -> Socket
	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: false,
			Type:      &SocketString,
			TypeIds:   IntToStringSlice(sockets),
			Scope:     scope,
		})
		return results, true
	}

	// Socket -> Node
	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
		results = append(results, ScopeQueryResult{
			Metric:    metric,
			Hostname:  hostname,
			Aggregate: true,
			Type:      &SocketString,
			TypeIds:   IntToStringSlice(sockets),
			Scope:     scope,
		})
		return results, true
	}

	// Node -> Node
	if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
		results = append(results, ScopeQueryResult{
			Metric:   metric,
			Hostname: hostname,
			Scope:    scope,
		})
		return results, true
	}

	// Unhandled Case
	return nil, false
}

// IntToStringSlice converts a slice of integers to a slice of strings.
// Used to convert hardware thread/core/socket IDs from topology (int) to query TypeIds (string).
// Optimized to reuse a byte buffer for string conversion, reducing allocations.
func IntToStringSlice(is []int) []string {
	if len(is) == 0 {
		return nil
	}

	ss := make([]string, len(is))
	buf := make([]byte, 0, 16) // Reusable buffer for integer conversion
	for i, x := range is {
		buf = strconv.AppendInt(buf[:0], int64(x), 10)
		ss[i] = string(buf)
	}
	return ss
}

// SanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
// If ANY of avg/min/max is NaN, ALL three are zeroed for consistency.
func SanitizeStats(avg, min, max *schema.Float) {
	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
		*avg = schema.Float(0)
		*min = schema.Float(0)
		*max = schema.Float(0)
	}
}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstoreclient - Query Building
     7→//
     8→// This file contains the query construction and scope transformation logic for cc-metric-store queries.
     9→// It handles the complex mapping between requested metric scopes and native hardware topology,
    10→// automatically aggregating or filtering metrics as needed.
    11→//
    12→// # Scope Transformations
    13→//
    14→// The buildScopeQueries function implements the core scope transformation algorithm.
    15→// It handles 25+ different transformation cases, mapping between:
    16→//   - Accelerator (GPU) scope
    17→//   - HWThread (hardware thread/SMT) scope
    18→//   - Core (CPU core) scope
    19→//   - Socket (CPU package) scope
    20→//   - MemoryDomain (NUMA domain) scope
    21→//   - Node (full system) scope
    22→//
    23→// Transformations follow these rules:
    24→//   - Same scope: Return data as-is (e.g., Core → Core)
    25→//   - Coarser scope: Aggregate data (e.g., Core → Socket with Aggregate=true)
    26→//   - Finer scope: Error - cannot increase granularity
    27→//
    28→// # Query Building
    29→//
    30→// buildQueries and buildNodeQueries are the main entry points, handling job-specific
    31→// and node-specific query construction respectively. They:
    32→//   - Validate metric configurations
    33→//   - Handle subcluster-specific metric filtering
    34→//   - Detect and skip duplicate scope requests
    35→//   - Call buildScopeQueries for each metric/scope/host combination
    36→package metricstoreclient
    37→
    38→import (
    39→	"fmt"
    40→
    41→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    42→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    43→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    44→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    45→)
    46→
    47→// buildQueries constructs API queries for job-specific metric data.
    48→// It iterates through metrics, scopes, and job resources to build the complete query set.
    49→//
    50→// The function handles:
    51→//   - Metric configuration validation and subcluster filtering
    52→//   - Scope deduplication to avoid redundant queries
    53→//   - Hardware thread list resolution (job-allocated vs full node)
    54→//   - Delegation to buildScopeQueries for scope transformations
    55→//
    56→// Returns queries and their corresponding assigned scopes (which may differ from requested scopes).
    57→func (ccms *CCMetricStore) buildQueries(
    58→	job *schema.Job,
    59→	metrics []string,
    60→	scopes []schema.MetricScope,
    61→	resolution int,
    62→) ([]APIQuery, []schema.MetricScope, error) {
    63→	// Initialize both slices together
    64→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
    65→	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(job.Resources))
    66→
    67→	topology, err := ccms.getTopology(job.Cluster, job.SubCluster)
    68→	if err != nil {
    69→		cclog.Errorf("could not load cluster %s subCluster %s topology: %s", job.Cluster, job.SubCluster, err.Error())
    70→		return nil, nil, err
    71→	}
    72→
    73→	for _, metric := range metrics {
    74→		remoteName := metric
    75→		mc := archive.GetMetricConfig(job.Cluster, metric)
    76→		if mc == nil {
    77→			cclog.Warnf("metric '%s' is not specified for cluster '%s' - skipping", metric, job.Cluster)
    78→			continue
    79→		}
    80→
    81→		// Skip if metric is removed for subcluster
    82→		if len(mc.SubClusters) != 0 {
    83→			isRemoved := false
    84→			for _, scConfig := range mc.SubClusters {
    85→				if scConfig.Name == job.SubCluster && scConfig.Remove {
    86→					isRemoved = true
    87→					break
    88→				}
    89→			}
    90→			if isRemoved {
    91→				continue
    92→			}
    93→		}
    94→
    95→		// Avoid duplicates...
    96→		handledScopes := make([]schema.MetricScope, 0, 3)
    97→
    98→	scopesLoop:
    99→		for _, requestedScope := range scopes {
   100→			nativeScope := mc.Scope
   101→			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
   102→				continue
   103→			}
   104→
   105→			scope := nativeScope.Max(requestedScope)
   106→			for _, s := range handledScopes {
   107→				if scope == s {
   108→					continue scopesLoop
   109→				}
   110→			}
   111→			handledScopes = append(handledScopes, scope)
   112→
   113→			for _, host := range job.Resources {
   114→				hwthreads := host.HWThreads
   115→				if hwthreads == nil {
   116→					hwthreads = topology.Node
   117→				}
   118→
   119→				scopeResults, ok := metricstore.BuildScopeQueries(
   120→					nativeScope, requestedScope,
   121→					remoteName, host.Hostname,
   122→					topology, hwthreads, host.Accelerators,
   123→				)
   124→
   125→				if !ok {
   126→					return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   127→				}
   128→
   129→				for _, sr := range scopeResults {
   130→					queries = append(queries, APIQuery{
   131→						Metric:     sr.Metric,
   132→						Hostname:   sr.Hostname,
   133→						Aggregate:  sr.Aggregate,
   134→						Type:       sr.Type,
   135→						TypeIds:    sr.TypeIds,
   136→						Resolution: resolution,
   137→					})
   138→					assignedScope = append(assignedScope, sr.Scope)
   139→				}
   140→			}
   141→		}
   142→	}
   143→
   144→	return queries, assignedScope, nil
   145→}
   146→
   147→// buildNodeQueries constructs API queries for node-specific metric data (Systems View).
   148→// Similar to buildQueries but uses full node topology instead of job-allocated resources.
   149→//
   150→// The function handles:
   151→//   - SubCluster topology resolution (either pre-loaded or per-node lookup)
   152→//   - Full node hardware thread lists (not job-specific subsets)
   153→//   - All accelerators on each node
   154→//   - Metric configuration validation with subcluster filtering
   155→//
   156→// Returns queries and their corresponding assigned scopes.
   157→func (ccms *CCMetricStore) buildNodeQueries(
   158→	cluster string,
   159→	subCluster string,
   160→	nodes []string,
   161→	metrics []string,
   162→	scopes []schema.MetricScope,
   163→	resolution int,
   164→) ([]APIQuery, []schema.MetricScope, error) {
   165→	// Initialize both slices together
   166→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
   167→	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(nodes))
   168→
   169→	for _, metric := range metrics {
   170→		remoteName := metric
   171→		mc := archive.GetMetricConfig(cluster, metric)
   172→		if mc == nil {
   173→			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
   174→			continue
   175→		}
   176→
   177→		// Skip if metric is removed for subcluster
   178→		if mc.SubClusters != nil {
   179→			isRemoved := false
   180→			for _, scConfig := range mc.SubClusters {
   181→				if scConfig.Name == subCluster && scConfig.Remove {
   182→					isRemoved = true
   183→					break
   184→				}
   185→			}
   186→			if isRemoved {
   187→				continue
   188→			}
   189→		}
   190→
   191→		// Avoid duplicates...
   192→		handledScopes := make([]schema.MetricScope, 0, 3)
   193→
   194→	scopesLoop:
   195→		for _, requestedScope := range scopes {
   196→			nativeScope := mc.Scope
   197→
   198→			scope := nativeScope.Max(requestedScope)
   199→			for _, s := range handledScopes {
   200→				if scope == s {
   201→					continue scopesLoop
   202→				}
   203→			}
   204→			handledScopes = append(handledScopes, scope)
   205→
   206→			for _, hostname := range nodes {
   207→				var topology *schema.Topology
   208→				var err error
   209→
   210→				// If no subCluster given, get it by node
   211→				if subCluster == "" {
   212→					topology, err = ccms.getTopologyByNode(cluster, hostname)
   213→				} else {
   214→					topology, err = ccms.getTopology(cluster, subCluster)
   215→				}
   216→
   217→				if err != nil {
   218→					return nil, nil, err
   219→				}
   220→
   221→				// Always full node hwthread id list, no partial queries expected -> Use "topology.Node" directly where applicable
   222→				// Always full accelerator id list, no partial queries expected -> Use "acceleratorIds" directly where applicable
   223→				acceleratorIds := topology.GetAcceleratorIDs()
   224→
   225→				// Moved check here if metric matches hardware specs
   226→				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
   227→					continue scopesLoop
   228→				}
   229→
   230→				scopeResults, ok := metricstore.BuildScopeQueries(
   231→					nativeScope, requestedScope,
   232→					remoteName, hostname,
   233→					topology, topology.Node, acceleratorIds,
   234→				)
   235→
   236→				if !ok {
   237→					return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   238→				}
   239→
   240→				for _, sr := range scopeResults {
   241→					queries = append(queries, APIQuery{
   242→						Metric:     sr.Metric,
   243→						Hostname:   sr.Hostname,
   244→						Aggregate:  sr.Aggregate,
   245→						Type:       sr.Type,
   246→						TypeIds:    sr.TypeIds,
   247→						Resolution: resolution,
   248→					})
   249→					assignedScope = append(assignedScope, sr.Scope)
   250→				}
   251→			}
   252→		}
   253→	}
   254→
   255→	return queries, assignedScope, nil
   256→}
   257→
   258→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstoreclient - Query Building
//
// This file contains the query construction and scope transformation logic for cc-metric-store queries.
// It handles the complex mapping between requested metric scopes and native hardware topology,
// automatically aggregating or filtering metrics as needed.
//
// # Scope Transformations
//
// The buildScopeQueries function implements the core scope transformation algorithm.
// It handles 25+ different transformation cases, mapping between:
//   - Accelerator (GPU) scope
//   - HWThread (hardware thread/SMT) scope
//   - Core (CPU core) scope
//   - Socket (CPU package) scope
//   - MemoryDomain (NUMA domain) scope
//   - Node (full system) scope
//
// Transformations follow these rules:
//   - Same scope: Return data as-is (e.g., Core → Core)
//   - Coarser scope: Aggregate data (e.g., Core → Socket with Aggregate=true)
//   - Finer scope: Error - cannot increase granularity
//
// # Query Building
//
// buildQueries and buildNodeQueries are the main entry points, handling job-specific
// and node-specific query construction respectively. They:
//   - Validate metric configurations
//   - Handle subcluster-specific metric filtering
//   - Detect and skip duplicate scope requests
//   - Call buildScopeQueries for each metric/scope/host combination
package metricstoreclient

import (
	"fmt"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// buildQueries constructs API queries for job-specific metric data.
// It iterates through metrics, scopes, and job resources to build the complete query set.
//
// The function handles:
//   - Metric configuration validation and subcluster filtering
//   - Scope deduplication to avoid redundant queries
//   - Hardware thread list resolution (job-allocated vs full node)
//   - Delegation to buildScopeQueries for scope transformations
//
// Returns queries and their corresponding assigned scopes (which may differ from requested scopes).
func (ccms *CCMetricStore) buildQueries(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
) ([]APIQuery, []schema.MetricScope, error) {
	// Initialize both slices together
	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(job.Resources))

	topology, err := ccms.getTopology(job.Cluster, job.SubCluster)
	if err != nil {
		cclog.Errorf("could not load cluster %s subCluster %s topology: %s", job.Cluster, job.SubCluster, err.Error())
		return nil, nil, err
	}

	for _, metric := range metrics {
		remoteName := metric
		mc := archive.GetMetricConfig(job.Cluster, metric)
		if mc == nil {
			cclog.Warnf("metric '%s' is not specified for cluster '%s' - skipping", metric, job.Cluster)
			continue
		}

		// Skip if metric is removed for subcluster
		if len(mc.SubClusters) != 0 {
			isRemoved := false
			for _, scConfig := range mc.SubClusters {
				if scConfig.Name == job.SubCluster && scConfig.Remove {
					isRemoved = true
					break
				}
			}
			if isRemoved {
				continue
			}
		}

		// Avoid duplicates...
		handledScopes := make([]schema.MetricScope, 0, 3)

	scopesLoop:
		for _, requestedScope := range scopes {
			nativeScope := mc.Scope
			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
				continue
			}

			scope := nativeScope.Max(requestedScope)
			for _, s := range handledScopes {
				if scope == s {
					continue scopesLoop
				}
			}
			handledScopes = append(handledScopes, scope)

			for _, host := range job.Resources {
				hwthreads := host.HWThreads
				if hwthreads == nil {
					hwthreads = topology.Node
				}

				scopeResults, ok := metricstore.BuildScopeQueries(
					nativeScope, requestedScope,
					remoteName, host.Hostname,
					topology, hwthreads, host.Accelerators,
				)

				if !ok {
					return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
				}

				for _, sr := range scopeResults {
					queries = append(queries, APIQuery{
						Metric:     sr.Metric,
						Hostname:   sr.Hostname,
						Aggregate:  sr.Aggregate,
						Type:       sr.Type,
						TypeIds:    sr.TypeIds,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, sr.Scope)
				}
			}
		}
	}

	return queries, assignedScope, nil
}

// buildNodeQueries constructs API queries for node-specific metric data (Systems View).
// Similar to buildQueries but uses full node topology instead of job-allocated resources.
//
// The function handles:
//   - SubCluster topology resolution (either pre-loaded or per-node lookup)
//   - Full node hardware thread lists (not job-specific subsets)
//   - All accelerators on each node
//   - Metric configuration validation with subcluster filtering
//
// Returns queries and their corresponding assigned scopes.
func (ccms *CCMetricStore) buildNodeQueries(
	cluster string,
	subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
) ([]APIQuery, []schema.MetricScope, error) {
	// Initialize both slices together
	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(nodes))

	for _, metric := range metrics {
		remoteName := metric
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
			continue
		}

		// Skip if metric is removed for subcluster
		if mc.SubClusters != nil {
			isRemoved := false
			for _, scConfig := range mc.SubClusters {
				if scConfig.Name == subCluster && scConfig.Remove {
					isRemoved = true
					break
				}
			}
			if isRemoved {
				continue
			}
		}

		// Avoid duplicates...
		handledScopes := make([]schema.MetricScope, 0, 3)

	scopesLoop:
		for _, requestedScope := range scopes {
			nativeScope := mc.Scope

			scope := nativeScope.Max(requestedScope)
			for _, s := range handledScopes {
				if scope == s {
					continue scopesLoop
				}
			}
			handledScopes = append(handledScopes, scope)

			for _, hostname := range nodes {
				var topology *schema.Topology
				var err error

				// If no subCluster given, get it by node
				if subCluster == "" {
					topology, err = ccms.getTopologyByNode(cluster, hostname)
				} else {
					topology, err = ccms.getTopology(cluster, subCluster)
				}

				if err != nil {
					return nil, nil, err
				}

				// Always full node hwthread id list, no partial queries expected -> Use "topology.Node" directly where applicable
				// Always full accelerator id list, no partial queries expected -> Use "acceleratorIds" directly where applicable
				acceleratorIds := topology.GetAcceleratorIDs()

				// Moved check here if metric matches hardware specs
				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
					continue scopesLoop
				}

				scopeResults, ok := metricstore.BuildScopeQueries(
					nativeScope, requestedScope,
					remoteName, hostname,
					topology, topology.Node, acceleratorIds,
				)

				if !ok {
					return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
				}

				for _, sr := range scopeResults {
					queries = append(queries, APIQuery{
						Metric:     sr.Metric,
						Hostname:   sr.Hostname,
						Aggregate:  sr.Aggregate,
						Type:       sr.Type,
						TypeIds:    sr.TypeIds,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, sr.Scope)
				}
			}
		}
	}

	return queries, assignedScope, nil
}