     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstoreclient provides a client for querying the cc-metric-store time series database.
     7→//
     8→// The cc-metric-store is a high-performance time series database optimized for HPC metric data.
     9→// This client handles HTTP communication, query construction, scope transformations, and data retrieval
    10→// for job and node metrics across different metric scopes (node, socket, core, hwthread, accelerator).
    11→//
    12→// # Architecture
    13→//
    14→// The package is split into two main components:
    15→//   - Client Operations (cc-metric-store.go): HTTP client, request handling, data loading methods
    16→//   - Query Building (cc-metric-store-queries.go): Query construction and scope transformation logic
    17→//
    18→// # Basic Usage
    19→//
    20→//	store := NewCCMetricStore("http://localhost:8080", "jwt-token")
    21→//
    22→//	// Load job data
    23→//	jobData, err := store.LoadData(job, metrics, scopes, ctx, resolution)
    24→//	if err != nil {
    25→//	    log.Fatal(err)
    26→//	}
    27→//
    28→// # Metric Scopes
    29→//
    30→// The client supports hierarchical metric scopes that map to HPC hardware topology:
    31→//   - MetricScopeAccelerator: GPU/accelerator level metrics
    32→//   - MetricScopeHWThread: Hardware thread (SMT) level metrics
    33→//   - MetricScopeCore: CPU core level metrics
    34→//   - MetricScopeSocket: CPU socket level metrics
    35→//   - MetricScopeMemoryDomain: NUMA domain level metrics
    36→//   - MetricScopeNode: Full node level metrics
    37→//
    38→// The client automatically handles scope transformations, aggregating finer-grained metrics
    39→// to coarser scopes when needed (e.g., aggregating core metrics to socket level).
    40→//
    41→// # Error Handling
    42→//
    43→// The client supports partial errors - if some queries fail, it returns both the successful
    44→// data and an error listing the failed queries. This allows processing partial results
    45→// when some nodes or metrics are temporarily unavailable.
    46→//
    47→// # API Versioning
    48→//
    49→// The client uses cc-metric-store API v2, which includes support for:
    50→//   - Data resampling for bandwidth optimization
    51→//   - Multi-scope queries in a single request
    52→//   - Aggregation across hardware topology levels
    53→package metricstoreclient
    54→
    55→import (
    56→	"bufio"
    57→	"bytes"
    58→	"context"
    59→	"encoding/json"
    60→	"fmt"
    61→	"net/http"
    62→	"strings"
    63→	"time"
    64→
    65→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    66→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    67→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    68→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    69→)
    70→
    71→// CCMetricStore is the HTTP client for communicating with cc-metric-store.
    72→// It manages connection details, authentication, and provides methods for querying metrics.
    73→type CCMetricStore struct {
    74→	client        http.Client                 // HTTP client with 10-second timeout
    75→	jwt           string                      // JWT Bearer token for authentication
    76→	url           string                      // Base URL of cc-metric-store instance
    77→	queryEndpoint string                      // Full URL to query API endpoint
    78→	topologyCache map[string]*schema.Topology // cluster -> topology cache
    79→}
    80→
    81→// APIQueryRequest represents a request to the cc-metric-store query API.
    82→// It supports both explicit queries and "for-all-nodes" bulk queries.
    83→type APIQueryRequest struct {
    84→	Cluster     string     `json:"cluster"`       // Target cluster name
    85→	Queries     []APIQuery `json:"queries"`       // Explicit list of metric queries
    86→	ForAllNodes []string   `json:"for-all-nodes"` // Metrics to query for all nodes
    87→	From        int64      `json:"from"`          // Start time (Unix timestamp)
    88→	To          int64      `json:"to"`            // End time (Unix timestamp)
    89→	WithStats   bool       `json:"with-stats"`    // Include min/avg/max statistics
    90→	WithData    bool       `json:"with-data"`     // Include time series data points
    91→}
    92→
    93→// APIQuery specifies a single metric query with optional scope filtering.
    94→// Type and TypeIds define the hardware scope (core, socket, accelerator, etc.).
    95→type APIQuery struct {
    96→	Type       *string  `json:"type,omitempty"`        // Scope type (e.g., "core", "socket")
    97→	SubType    *string  `json:"subtype,omitempty"`     // Sub-scope type (reserved for future use)
    98→	Metric     string   `json:"metric"`                // Metric name
    99→	Hostname   string   `json:"host"`                  // Target hostname
   100→	Resolution int      `json:"resolution"`            // Data resolution in seconds (0 = native)
   101→	TypeIds    []string `json:"type-ids,omitempty"`    // IDs for the scope type (e.g., core IDs)
   102→	SubTypeIds []string `json:"subtype-ids,omitempty"` // IDs for sub-scope (reserved)
   103→	Aggregate  bool     `json:"aggreg"`                // Aggregate across TypeIds
   104→}
   105→
   106→// APIQueryResponse contains the results from a cc-metric-store query.
   107→// Results align with the Queries slice by index.
   108→type APIQueryResponse struct {
   109→	Queries []APIQuery        `json:"queries,omitempty"` // Echoed queries (for bulk requests)
   110→	Results [][]APIMetricData `json:"results"`           // Result data, indexed by query
   111→}
   112→
   113→// APIMetricData represents time series data and statistics for a single metric series.
   114→// Error is set if this particular series failed to load.
   115→type APIMetricData struct {
   116→	Error      *string        `json:"error"`      // Error message if query failed
   117→	Data       []schema.Float `json:"data"`       // Time series data points
   118→	From       int64          `json:"from"`       // Actual start time of data
   119→	To         int64          `json:"to"`         // Actual end time of data
   120→	Resolution int            `json:"resolution"` // Actual resolution of data in seconds
   121→	Avg        schema.Float   `json:"avg"`        // Average value across time range
   122→	Min        schema.Float   `json:"min"`        // Minimum value in time range
   123→	Max        schema.Float   `json:"max"`        // Maximum value in time range
   124→}
   125→
   126→// NewCCMetricStore creates and initializes a new (external) CCMetricStore client.
   127→// The url parameter should include the protocol and port (e.g., "http://localhost:8080").
   128→// The token parameter is a JWT used for Bearer authentication; pass empty string if auth is disabled.
   129→func NewCCMetricStore(url string, token string) *CCMetricStore {
   130→	return &CCMetricStore{
   131→		url:           url,
   132→		queryEndpoint: fmt.Sprintf("%s/api/query", url),
   133→		jwt:           token,
   134→		client: http.Client{
   135→			Timeout: 10 * time.Second,
   136→		},
   137→		topologyCache: make(map[string]*schema.Topology),
   138→	}
   139→}
   140→
   141→// doRequest executes an HTTP POST request to the cc-metric-store query API.
   142→// It handles JSON encoding/decoding, authentication, and API versioning.
   143→// The request body is automatically closed to prevent resource leaks.
   144→func (ccms *CCMetricStore) doRequest(
   145→	ctx context.Context,
   146→	body *APIQueryRequest,
   147→) (*APIQueryResponse, error) {
   148→	buf := &bytes.Buffer{}
   149→	if err := json.NewEncoder(buf).Encode(body); err != nil {
   150→		cclog.Errorf("Error while encoding request body: %s", err.Error())
   151→		return nil, err
   152→	}
   153→
   154→	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf)
   155→	if err != nil {
   156→		cclog.Errorf("Error while building request body: %s", err.Error())
   157→		return nil, err
   158→	}
   159→	if ccms.jwt != "" {
   160→		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   161→	}
   162→
   163→	// versioning the cc-metric-store query API.
   164→	// v2 = data with resampling
   165→	// v1 = data without resampling
   166→	q := req.URL.Query()
   167→	q.Add("version", "v2")
   168→	req.URL.RawQuery = q.Encode()
   169→
   170→	res, err := ccms.client.Do(req)
   171→	if err != nil {
   172→		cclog.Errorf("Error while performing request: %s", err.Error())
   173→		return nil, err
   174→	}
   175→	defer res.Body.Close()
   176→
   177→	if res.StatusCode != http.StatusOK {
   178→		return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status)
   179→	}
   180→
   181→	var resBody APIQueryResponse
   182→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
   183→		cclog.Errorf("Error while decoding result body: %s", err.Error())
   184→		return nil, err
   185→	}
   186→
   187→	return &resBody, nil
   188→}
   189→
   190→// getTopology returns the topology for a given cluster and subcluster, caching it if not already present
   191→func (ccms *CCMetricStore) getTopology(cluster, subCluster string) (*schema.Topology, error) {
   192→	cacheKey := fmt.Sprintf("%s:%s", cluster, subCluster)
   193→	if topology, ok := ccms.topologyCache[cacheKey]; ok {
   194→		return topology, nil
   195→	}
   196→
   197→	subcluster, err := archive.GetSubCluster(cluster, subCluster)
   198→	if err != nil {
   199→		return nil, err
   200→	}
   201→
   202→	ccms.topologyCache[cacheKey] = &subcluster.Topology
   203→	return &subcluster.Topology, nil
   204→}
   205→
   206→// getTopologyByNode returns the topology for a given cluster and node, caching it if not already present
   207→func (ccms *CCMetricStore) getTopologyByNode(cluster, node string) (*schema.Topology, error) {
   208→	subCluster, err := archive.GetSubClusterByNode(cluster, node)
   209→	if err != nil {
   210→		return nil, err
   211→	}
   212→
   213→	return ccms.getTopology(cluster, subCluster)
   214→}
   215→
   216→// LoadData retrieves time series data and statistics for the specified job and metrics.
   217→// It queries data for the job's time range and resources, handling scope transformations automatically.
   218→//
   219→// Parameters:
   220→//   - job: Job metadata including cluster, time range, and allocated resources
   221→//   - metrics: List of metric names to retrieve
   222→//   - scopes: Requested metric scopes (node, socket, core, etc.)
   223→//   - ctx: Context for cancellation and timeouts
   224→//   - resolution: Data resolution in seconds (0 for native resolution)
   225→//
   226→// Returns JobData organized as: metric -> scope -> series list.
   227→// Supports partial errors: returns available data even if some queries fail.
   228→func (ccms *CCMetricStore) LoadData(
   229→	job *schema.Job,
   230→	metrics []string,
   231→	scopes []schema.MetricScope,
   232→	ctx context.Context,
   233→	resolution int,
   234→) (schema.JobData, error) {
   235→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
   236→	if err != nil {
   237→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   238→		return nil, err
   239→	}
   240→
   241→	// Verify assignment is correct - log any inconsistencies for debugging
   242→	if len(queries) != len(assignedScope) {
   243→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
   244→			len(queries), len(assignedScope))
   245→	}
   246→
   247→	req := APIQueryRequest{
   248→		Cluster:   job.Cluster,
   249→		From:      job.StartTime,
   250→		To:        job.StartTime + int64(job.Duration),
   251→		Queries:   queries,
   252→		WithStats: true,
   253→		WithData:  true,
   254→	}
   255→
   256→	resBody, err := ccms.doRequest(ctx, &req)
   257→	if err != nil {
   258→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   259→		return nil, err
   260→	}
   261→
   262→	var errors []string
   263→	jobData := make(schema.JobData)
   264→
   265→	// Add safety check for potential index out of range errors
   266→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   267→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   268→			len(req.Queries), len(resBody.Results), len(assignedScope))
   269→		if len(resBody.Results) > len(req.Queries) {
   270→			resBody.Results = resBody.Results[:len(req.Queries)]
   271→		}
   272→		if len(assignedScope) > len(req.Queries) {
   273→			assignedScope = assignedScope[:len(req.Queries)]
   274→		}
   275→	}
   276→
   277→	for i, row := range resBody.Results {
   278→		// Safety check to prevent index out of range errors
   279→		if i >= len(req.Queries) || i >= len(assignedScope) {
   280→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   281→				i, len(req.Queries), len(assignedScope))
   282→			continue
   283→		}
   284→
   285→		query := req.Queries[i]
   286→		metric := query.Metric
   287→		scope := assignedScope[i]
   288→		mc := archive.GetMetricConfig(job.Cluster, metric)
   289→
   290→		if mc == nil {
   291→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   292→			continue
   293→		}
   294→
   295→		if _, ok := jobData[metric]; !ok {
   296→			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
   297→		}
   298→
   299→		res := mc.Timestep
   300→		if len(row) > 0 {
   301→			res = row[0].Resolution
   302→		}
   303→
   304→		jobMetric, ok := jobData[metric][scope]
   305→		if !ok {
   306→			jobMetric = &schema.JobMetric{
   307→				Unit:     mc.Unit,
   308→				Timestep: res,
   309→				Series:   make([]schema.Series, 0),
   310→			}
   311→			jobData[metric][scope] = jobMetric
   312→		}
   313→
   314→		for ndx, res := range row {
   315→			if res.Error != nil {
   316→				/* Build list for "partial errors", if any */
   317→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   318→				continue
   319→			}
   320→
   321→			id := (*string)(nil)
   322→			if query.Type != nil {
   323→				// Check if ndx is within the bounds of TypeIds slice
   324→				if ndx < len(query.TypeIds) {
   325→					id = new(string)
   326→					*id = query.TypeIds[ndx]
   327→				} else {
   328→					// Log the error but continue processing
   329→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   330→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   331→				}
   332→			}
   333→
   334→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   335→
   336→			jobMetric.Series = append(jobMetric.Series, schema.Series{
   337→				Hostname: query.Hostname,
   338→				ID:       id,
   339→				Statistics: schema.MetricStatistics{
   340→					Avg: float64(res.Avg),
   341→					Min: float64(res.Min),
   342→					Max: float64(res.Max),
   343→				},
   344→				Data: res.Data,
   345→			})
   346→		}
   347→
   348→		// So that one can later check len(jobData):
   349→		if len(jobMetric.Series) == 0 {
   350→			delete(jobData[metric], scope)
   351→			if len(jobData[metric]) == 0 {
   352→				delete(jobData, metric)
   353→			}
   354→		}
   355→	}
   356→
   357→	if len(errors) != 0 {
   358→		/* Returns list for "partial errors" */
   359→		return jobData, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   360→	}
   361→	return jobData, nil
   362→}
   363→
   364→// LoadStats retrieves min/avg/max statistics for job metrics at node scope.
   365→// This is faster than LoadData when only statistical summaries are needed (no time series data).
   366→//
   367→// Returns statistics organized as: metric -> hostname -> statistics.
   368→func (ccms *CCMetricStore) LoadStats(
   369→	job *schema.Job,
   370→	metrics []string,
   371→	ctx context.Context,
   372→) (map[string]map[string]schema.MetricStatistics, error) {
   373→	queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization?
   374→	if err != nil {
   375→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
   376→		return nil, err
   377→	}
   378→
   379→	req := APIQueryRequest{
   380→		Cluster:   job.Cluster,
   381→		From:      job.StartTime,
   382→		To:        job.StartTime + int64(job.Duration),
   383→		Queries:   queries,
   384→		WithStats: true,
   385→		WithData:  false,
   386→	}
   387→
   388→	resBody, err := ccms.doRequest(ctx, &req)
   389→	if err != nil {
   390→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   391→		return nil, err
   392→	}
   393→
   394→	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
   395→	for i, res := range resBody.Results {
   396→		if len(res) == 0 {
   397→			// No Data Found For Metric, Logged in FetchData to Warn
   398→			continue
   399→		}
   400→		query := req.Queries[i]
   401→		metric := query.Metric
   402→		data := res[0]
   403→		if data.Error != nil {
   404→			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
   405→			continue
   406→		}
   407→
   408→		metricdata, ok := stats[metric]
   409→		if !ok {
   410→			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
   411→			stats[metric] = metricdata
   412→		}
   413→
   414→		if hasNaNStats(data.Avg, data.Min, data.Max) {
   415→			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
   416→			continue
   417→		}
   418→
   419→		metricdata[query.Hostname] = schema.MetricStatistics{
   420→			Avg: float64(data.Avg),
   421→			Min: float64(data.Min),
   422→			Max: float64(data.Max),
   423→		}
   424→	}
   425→
   426→	return stats, nil
   427→}
   428→
   429→// LoadScopedStats retrieves statistics for job metrics across multiple scopes.
   430→// Used for the Job-View Statistics Table to display per-scope breakdowns.
   431→//
   432→// Returns statistics organized as: metric -> scope -> list of scoped statistics.
   433→// Each scoped statistic includes hostname, hardware ID (if applicable), and min/avg/max values.
   434→func (ccms *CCMetricStore) LoadScopedStats(
   435→	job *schema.Job,
   436→	metrics []string,
   437→	scopes []schema.MetricScope,
   438→	ctx context.Context,
   439→) (schema.ScopedJobStats, error) {
   440→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
   441→	if err != nil {
   442→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   443→		return nil, err
   444→	}
   445→
   446→	req := APIQueryRequest{
   447→		Cluster:   job.Cluster,
   448→		From:      job.StartTime,
   449→		To:        job.StartTime + int64(job.Duration),
   450→		Queries:   queries,
   451→		WithStats: true,
   452→		WithData:  false,
   453→	}
   454→
   455→	resBody, err := ccms.doRequest(ctx, &req)
   456→	if err != nil {
   457→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   458→		return nil, err
   459→	}
   460→
   461→	var errors []string
   462→	scopedJobStats := make(schema.ScopedJobStats)
   463→
   464→	for i, row := range resBody.Results {
   465→		query := req.Queries[i]
   466→		metric := query.Metric
   467→		scope := assignedScope[i]
   468→
   469→		if _, ok := scopedJobStats[metric]; !ok {
   470→			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
   471→		}
   472→
   473→		if _, ok := scopedJobStats[metric][scope]; !ok {
   474→			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
   475→		}
   476→
   477→		for ndx, res := range row {
   478→			if res.Error != nil {
   479→				/* Build list for "partial errors", if any */
   480→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   481→				continue
   482→			}
   483→
   484→			id := (*string)(nil)
   485→			if query.Type != nil {
   486→				// Check if ndx is within the bounds of TypeIds slice
   487→				if ndx < len(query.TypeIds) {
   488→					id = new(string)
   489→					*id = query.TypeIds[ndx]
   490→				} else {
   491→					// Log the error but continue processing
   492→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   493→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   494→				}
   495→			}
   496→
   497→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   498→
   499→			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
   500→				Hostname: query.Hostname,
   501→				ID:       id,
   502→				Data: &schema.MetricStatistics{
   503→					Avg: float64(res.Avg),
   504→					Min: float64(res.Min),
   505→					Max: float64(res.Max),
   506→				},
   507→			})
   508→		}
   509→
   510→		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
   511→		if len(scopedJobStats[metric][scope]) == 0 {
   512→			delete(scopedJobStats[metric], scope)
   513→			if len(scopedJobStats[metric]) == 0 {
   514→				delete(scopedJobStats, metric)
   515→			}
   516→		}
   517→	}
   518→
   519→	if len(errors) != 0 {
   520→		/* Returns list for "partial errors" */
   521→		return scopedJobStats, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   522→	}
   523→	return scopedJobStats, nil
   524→}
   525→
   526→// LoadNodeData retrieves current metric data for specified nodes in a cluster.
   527→// Used for the Systems-View Node-Overview to display real-time node status.
   528→//
   529→// If nodes is nil, queries all metrics for all nodes in the cluster (bulk query).
   530→// Returns data organized as: hostname -> metric -> list of JobMetric (with time series and stats).
   531→func (ccms *CCMetricStore) LoadNodeData(
   532→	cluster string,
   533→	metrics, nodes []string,
   534→	scopes []schema.MetricScope,
   535→	from, to time.Time,
   536→	ctx context.Context,
   537→) (map[string]map[string][]*schema.JobMetric, error) {
   538→	req := APIQueryRequest{
   539→		Cluster:   cluster,
   540→		From:      from.Unix(),
   541→		To:        to.Unix(),
   542→		WithStats: true,
   543→		WithData:  true,
   544→	}
   545→
   546→	if nodes == nil {
   547→		req.ForAllNodes = append(req.ForAllNodes, metrics...)
   548→	} else {
   549→		for _, node := range nodes {
   550→			for _, metric := range metrics {
   551→				req.Queries = append(req.Queries, APIQuery{
   552→					Hostname:   node,
   553→					Metric:     metric,
   554→					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
   555→				})
   556→			}
   557→		}
   558→	}
   559→
   560→	resBody, err := ccms.doRequest(ctx, &req)
   561→	if err != nil {
   562→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   563→		return nil, err
   564→	}
   565→
   566→	var errors []string
   567→	data := make(map[string]map[string][]*schema.JobMetric)
   568→	for i, res := range resBody.Results {
   569→		if len(res) == 0 {
   570→			// No Data Found For Metric, Logged in FetchData to Warn
   571→			continue
   572→		}
   573→
   574→		var query APIQuery
   575→		if resBody.Queries != nil {
   576→			query = resBody.Queries[i]
   577→		} else {
   578→			query = req.Queries[i]
   579→		}
   580→
   581→		metric := query.Metric
   582→		qdata := res[0]
   583→		if qdata.Error != nil {
   584→			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
   585→		}
   586→
   587→		sanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)
   588→
   589→		hostdata, ok := data[query.Hostname]
   590→		if !ok {
   591→			hostdata = make(map[string][]*schema.JobMetric)
   592→			data[query.Hostname] = hostdata
   593→		}
   594→
   595→		mc := archive.GetMetricConfig(cluster, metric)
   596→		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
   597→			Unit:     mc.Unit,
   598→			Timestep: mc.Timestep,
   599→			Series: []schema.Series{
   600→				{
   601→					Hostname: query.Hostname,
   602→					Data:     qdata.Data,
   603→					Statistics: schema.MetricStatistics{
   604→						Avg: float64(qdata.Avg),
   605→						Min: float64(qdata.Min),
   606→						Max: float64(qdata.Max),
   607→					},
   608→				},
   609→			},
   610→		})
   611→	}
   612→
   613→	if len(errors) != 0 {
   614→		/* Returns list of "partial errors" */
   615→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   616→	}
   617→
   618→	return data, nil
   619→}
   620→
   621→// LoadNodeListData retrieves paginated node metrics for the Systems-View Node-List.
   622→//
   623→// Supports filtering by subcluster and node name pattern. The nodeFilter performs
   624→// substring matching on hostnames.
   625→//
   626→// Returns:
   627→//   - Node data organized as: hostname -> JobData (metric -> scope -> series)
   628→//   - Total node count (before pagination)
   629→//   - HasNextPage flag indicating if more pages are available
   630→//   - Error (may be partial error with some data returned)
   631→func (ccms *CCMetricStore) LoadNodeListData(
   632→	cluster, subCluster string,
   633→	nodes []string,
   634→	metrics []string,
   635→	scopes []schema.MetricScope,
   636→	resolution int,
   637→	from, to time.Time,
   638→	ctx context.Context,
   639→) (map[string]schema.JobData, error) {
   640→	queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
   641→	if err != nil {
   642→		cclog.Errorf("Error while building node queries for Cluster %s, SubCluster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
   643→		return nil, err
   644→	}
   645→
   646→	// Verify assignment is correct - log any inconsistencies for debugging
   647→	if len(queries) != len(assignedScope) {
   648→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
   649→			len(queries), len(assignedScope))
   650→	}
   651→
   652→	req := APIQueryRequest{
   653→		Cluster:   cluster,
   654→		Queries:   queries,
   655→		From:      from.Unix(),
   656→		To:        to.Unix(),
   657→		WithStats: true,
   658→		WithData:  true,
   659→	}
   660→
   661→	resBody, err := ccms.doRequest(ctx, &req)
   662→	if err != nil {
   663→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   664→		return nil, err
   665→	}
   666→
   667→	var errors []string
   668→	data := make(map[string]schema.JobData)
   669→
   670→	// Add safety check for index out of range issues
   671→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   672→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   673→			len(req.Queries), len(resBody.Results), len(assignedScope))
   674→		if len(resBody.Results) > len(req.Queries) {
   675→			resBody.Results = resBody.Results[:len(req.Queries)]
   676→		}
   677→		if len(assignedScope) > len(req.Queries) {
   678→			assignedScope = assignedScope[:len(req.Queries)]
   679→		}
   680→	}
   681→
   682→	for i, row := range resBody.Results {
   683→		// Safety check to prevent index out of range errors
   684→		if i >= len(req.Queries) || i >= len(assignedScope) {
   685→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   686→				i, len(req.Queries), len(assignedScope))
   687→			continue
   688→		}
   689→
   690→		var query APIQuery
   691→		if resBody.Queries != nil {
   692→			if i < len(resBody.Queries) {
   693→				query = resBody.Queries[i]
   694→			} else {
   695→				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
   696→					i, len(resBody.Queries))
   697→				continue
   698→			}
   699→		} else {
   700→			query = req.Queries[i]
   701→		}
   702→
   703→		metric := query.Metric
   704→		scope := assignedScope[i]
   705→		mc := archive.GetMetricConfig(cluster, metric)
   706→		if mc == nil {
   707→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   708→			continue
   709→		}
   710→
   711→		res := mc.Timestep
   712→		if len(row) > 0 {
   713→			res = row[0].Resolution
   714→		}
   715→
   716→		// Init Nested Map Data Structures If Not Found
   717→		hostData, ok := data[query.Hostname]
   718→		if !ok {
   719→			hostData = make(schema.JobData)
   720→			data[query.Hostname] = hostData
   721→		}
   722→
   723→		metricData, ok := hostData[metric]
   724→		if !ok {
   725→			metricData = make(map[schema.MetricScope]*schema.JobMetric)
   726→			data[query.Hostname][metric] = metricData
   727→		}
   728→
   729→		scopeData, ok := metricData[scope]
   730→		if !ok {
   731→			scopeData = &schema.JobMetric{
   732→				Unit:     mc.Unit,
   733→				Timestep: res,
   734→				Series:   make([]schema.Series, 0),
   735→			}
   736→			data[query.Hostname][metric][scope] = scopeData
   737→		}
   738→
   739→		for ndx, res := range row {
   740→			if res.Error != nil {
   741→				/* Build list for "partial errors", if any */
   742→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   743→				continue
   744→			}
   745→
   746→			id := (*string)(nil)
   747→			if query.Type != nil {
   748→				// Check if ndx is within the bounds of TypeIds slice
   749→				if ndx < len(query.TypeIds) {
   750→					id = new(string)
   751→					*id = query.TypeIds[ndx]
   752→				} else {
   753→					// Log the error but continue processing
   754→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   755→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   756→				}
   757→			}
   758→
   759→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   760→
   761→			scopeData.Series = append(scopeData.Series, schema.Series{
   762→				Hostname: query.Hostname,
   763→				ID:       id,
   764→				Statistics: schema.MetricStatistics{
   765→					Avg: float64(res.Avg),
   766→					Min: float64(res.Min),
   767→					Max: float64(res.Max),
   768→				},
   769→				Data: res.Data,
   770→			})
   771→		}
   772→	}
   773→
   774→	if len(errors) != 0 {
   775→		/* Returns list of "partial errors" */
   776→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   777→	}
   778→
   779→	return data, nil
   780→}
   781→
   782→// HealthCheck queries the external cc-metric-store's health check endpoint.
   783→// It sends a HealthCheckReq as the request body to /api/healthcheck and
   784→// returns the per-node health check results.
   785→func (ccms *CCMetricStore) HealthCheck(cluster string,
   786→	nodes []string, metrics []string,
   787→) (map[string]metricstore.HealthCheckResult, error) {
   788→	req := metricstore.HealthCheckReq{
   789→		Cluster:     cluster,
   790→		Nodes:       nodes,
   791→		MetricNames: metrics,
   792→	}
   793→
   794→	buf := &bytes.Buffer{}
   795→	if err := json.NewEncoder(buf).Encode(req); err != nil {
   796→		cclog.Errorf("Error while encoding health check request body: %s", err.Error())
   797→		return nil, err
   798→	}
   799→
   800→	endpoint := fmt.Sprintf("%s/api/healthcheck", ccms.url)
   801→	httpReq, err := http.NewRequest(http.MethodGet, endpoint, buf)
   802→	if err != nil {
   803→		cclog.Errorf("Error while building health check request: %s", err.Error())
   804→		return nil, err
   805→	}
   806→	if ccms.jwt != "" {
   807→		httpReq.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   808→	}
   809→
   810→	res, err := ccms.client.Do(httpReq)
   811→	if err != nil {
   812→		cclog.Errorf("Error while performing health check request: %s", err.Error())
   813→		return nil, err
   814→	}
   815→	defer res.Body.Close()
   816→
   817→	if res.StatusCode != http.StatusOK {
   818→		return nil, fmt.Errorf("'%s': HTTP Status: %s", endpoint, res.Status)
   819→	}
   820→
   821→	var results map[string]metricstore.HealthCheckResult
   822→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&results); err != nil {
   823→		cclog.Errorf("Error while decoding health check response: %s", err.Error())
   824→		return nil, err
   825→	}
   826→
   827→	return results, nil
   828→}
   829→
   830→// sanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
   831→// Regular float64 values cannot be JSONed when NaN.
   832→func sanitizeStats(avg, min, max *schema.Float) {
   833→	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
   834→		*avg = schema.Float(0)
   835→		*min = schema.Float(0)
   836→		*max = schema.Float(0)
   837→	}
   838→}
   839→
   840→// hasNaNStats returns true if any of the statistics contain NaN values.
   841→func hasNaNStats(avg, min, max schema.Float) bool {
   842→	return avg.IsNaN() || min.IsNaN() || max.IsNaN()
   843→}
   844→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricstoreclient provides a client for querying the cc-metric-store time series database.
//
// The cc-metric-store is a high-performance time series database optimized for HPC metric data.
// This client handles HTTP communication, query construction, scope transformations, and data retrieval
// for job and node metrics across different metric scopes (node, socket, core, hwthread, accelerator).
//
// # Architecture
//
// The package is split into two main components:
//   - Client Operations (cc-metric-store.go): HTTP client, request handling, data loading methods
//   - Query Building (cc-metric-store-queries.go): Query construction and scope transformation logic
//
// # Basic Usage
//
//	store := NewCCMetricStore("http://localhost:8080", "jwt-token")
//
//	// Load job data
//	jobData, err := store.LoadData(job, metrics, scopes, ctx, resolution)
//	if err != nil {
//	    log.Fatal(err)
//	}
//
// # Metric Scopes
//
// The client supports hierarchical metric scopes that map to HPC hardware topology:
//   - MetricScopeAccelerator: GPU/accelerator level metrics
//   - MetricScopeHWThread: Hardware thread (SMT) level metrics
//   - MetricScopeCore: CPU core level metrics
//   - MetricScopeSocket: CPU socket level metrics
//   - MetricScopeMemoryDomain: NUMA domain level metrics
//   - MetricScopeNode: Full node level metrics
//
// The client automatically handles scope transformations, aggregating finer-grained metrics
// to coarser scopes when needed (e.g., aggregating core metrics to socket level).
//
// # Error Handling
//
// The client supports partial errors - if some queries fail, it returns both the successful
// data and an error listing the failed queries. This allows processing partial results
// when some nodes or metrics are temporarily unavailable.
//
// # API Versioning
//
// The client uses cc-metric-store API v2, which includes support for:
//   - Data resampling for bandwidth optimization
//   - Multi-scope queries in a single request
//   - Aggregation across hardware topology levels
package metricstoreclient

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// CCMetricStore is the HTTP client for communicating with cc-metric-store.
// It manages connection details, authentication, and provides methods for querying metrics.
type CCMetricStore struct {
	client        http.Client                 // HTTP client with 10-second timeout
	jwt           string                      // JWT Bearer token for authentication
	url           string                      // Base URL of cc-metric-store instance
	queryEndpoint string                      // Full URL to query API endpoint
	topologyCache map[string]*schema.Topology // cluster -> topology cache
}

// APIQueryRequest represents a request to the cc-metric-store query API.
// It supports both explicit queries and "for-all-nodes" bulk queries.
type APIQueryRequest struct {
	Cluster     string     `json:"cluster"`       // Target cluster name
	Queries     []APIQuery `json:"queries"`       // Explicit list of metric queries
	ForAllNodes []string   `json:"for-all-nodes"` // Metrics to query for all nodes
	From        int64      `json:"from"`          // Start time (Unix timestamp)
	To          int64      `json:"to"`            // End time (Unix timestamp)
	WithStats   bool       `json:"with-stats"`    // Include min/avg/max statistics
	WithData    bool       `json:"with-data"`     // Include time series data points
}

// APIQuery specifies a single metric query with optional scope filtering.
// Type and TypeIds define the hardware scope (core, socket, accelerator, etc.).
type APIQuery struct {
	Type       *string  `json:"type,omitempty"`        // Scope type (e.g., "core", "socket")
	SubType    *string  `json:"subtype,omitempty"`     // Sub-scope type (reserved for future use)
	Metric     string   `json:"metric"`                // Metric name
	Hostname   string   `json:"host"`                  // Target hostname
	Resolution int      `json:"resolution"`            // Data resolution in seconds (0 = native)
	TypeIds    []string `json:"type-ids,omitempty"`    // IDs for the scope type (e.g., core IDs)
	SubTypeIds []string `json:"subtype-ids,omitempty"` // IDs for sub-scope (reserved)
	Aggregate  bool     `json:"aggreg"`                // Aggregate across TypeIds
}

// APIQueryResponse contains the results from a cc-metric-store query.
// Results align with the Queries slice by index.
type APIQueryResponse struct {
	Queries []APIQuery        `json:"queries,omitempty"` // Echoed queries (for bulk requests)
	Results [][]APIMetricData `json:"results"`           // Result data, indexed by query
}

// APIMetricData represents time series data and statistics for a single metric series.
// Error is set if this particular series failed to load.
type APIMetricData struct {
	Error      *string        `json:"error"`      // Error message if query failed
	Data       []schema.Float `json:"data"`       // Time series data points
	From       int64          `json:"from"`       // Actual start time of data
	To         int64          `json:"to"`         // Actual end time of data
	Resolution int            `json:"resolution"` // Actual resolution of data in seconds
	Avg        schema.Float   `json:"avg"`        // Average value across time range
	Min        schema.Float   `json:"min"`        // Minimum value in time range
	Max        schema.Float   `json:"max"`        // Maximum value in time range
}

// NewCCMetricStore creates and initializes a new (external) CCMetricStore client.
// The url parameter should include the protocol and port (e.g., "http://localhost:8080").
// The token parameter is a JWT used for Bearer authentication; pass empty string if auth is disabled.
func NewCCMetricStore(url string, token string) *CCMetricStore {
	return &CCMetricStore{
		url:           url,
		queryEndpoint: fmt.Sprintf("%s/api/query", url),
		jwt:           token,
		client: http.Client{
			Timeout: 10 * time.Second,
		},
		topologyCache: make(map[string]*schema.Topology),
	}
}

// doRequest executes an HTTP POST request to the cc-metric-store query API.
// It handles JSON encoding/decoding, authentication, and API versioning.
// The request body is automatically closed to prevent resource leaks.
func (ccms *CCMetricStore) doRequest(
	ctx context.Context,
	body *APIQueryRequest,
) (*APIQueryResponse, error) {
	buf := &bytes.Buffer{}
	if err := json.NewEncoder(buf).Encode(body); err != nil {
		cclog.Errorf("Error while encoding request body: %s", err.Error())
		return nil, err
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf)
	if err != nil {
		cclog.Errorf("Error while building request body: %s", err.Error())
		return nil, err
	}
	if ccms.jwt != "" {
		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
	}

	// versioning the cc-metric-store query API.
	// v2 = data with resampling
	// v1 = data without resampling
	q := req.URL.Query()
	q.Add("version", "v2")
	req.URL.RawQuery = q.Encode()

	res, err := ccms.client.Do(req)
	if err != nil {
		cclog.Errorf("Error while performing request: %s", err.Error())
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status)
	}

	var resBody APIQueryResponse
	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
		cclog.Errorf("Error while decoding result body: %s", err.Error())
		return nil, err
	}

	return &resBody, nil
}

// getTopology returns the topology for a given cluster and subcluster, caching it if not already present
func (ccms *CCMetricStore) getTopology(cluster, subCluster string) (*schema.Topology, error) {
	cacheKey := fmt.Sprintf("%s:%s", cluster, subCluster)
	if topology, ok := ccms.topologyCache[cacheKey]; ok {
		return topology, nil
	}

	subcluster, err := archive.GetSubCluster(cluster, subCluster)
	if err != nil {
		return nil, err
	}

	ccms.topologyCache[cacheKey] = &subcluster.Topology
	return &subcluster.Topology, nil
}

// getTopologyByNode returns the topology for a given cluster and node, caching it if not already present
func (ccms *CCMetricStore) getTopologyByNode(cluster, node string) (*schema.Topology, error) {
	subCluster, err := archive.GetSubClusterByNode(cluster, node)
	if err != nil {
		return nil, err
	}

	return ccms.getTopology(cluster, subCluster)
}

// LoadData retrieves time series data and statistics for the specified job and metrics.
// It queries data for the job's time range and resources, handling scope transformations automatically.
//
// Parameters:
//   - job: Job metadata including cluster, time range, and allocated resources
//   - metrics: List of metric names to retrieve
//   - scopes: Requested metric scopes (node, socket, core, etc.)
//   - ctx: Context for cancellation and timeouts
//   - resolution: Data resolution in seconds (0 for native resolution)
//
// Returns JobData organized as: metric -> scope -> series list.
// Supports partial errors: returns available data even if some queries fail.
func (ccms *CCMetricStore) LoadData(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
	resolution int,
) (schema.JobData, error) {
	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  true,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	var errors []string
	jobData := make(schema.JobData)

	// Add safety check for potential index out of range errors
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		// Safety check to prevent index out of range errors
		if i >= len(req.Queries) || i >= len(assignedScope) {
			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
				i, len(req.Queries), len(assignedScope))
			continue
		}

		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(job.Cluster, metric)

		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
			continue
		}

		if _, ok := jobData[metric]; !ok {
			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = row[0].Resolution
		}

		jobMetric, ok := jobData[metric][scope]
		if !ok {
			jobMetric = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			jobData[metric][scope] = jobMetric
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			sanitizeStats(&res.Avg, &res.Min, &res.Max)

			jobMetric.Series = append(jobMetric.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}

		// So that one can later check len(jobData):
		if len(jobMetric.Series) == 0 {
			delete(jobData[metric], scope)
			if len(jobData[metric]) == 0 {
				delete(jobData, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return jobData, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return jobData, nil
}

// LoadStats retrieves min/avg/max statistics for job metrics at node scope.
// This is faster than LoadData when only statistical summaries are needed (no time series data).
//
// Returns statistics organized as: metric -> hostname -> statistics.
func (ccms *CCMetricStore) LoadStats(
	job *schema.Job,
	metrics []string,
	ctx context.Context,
) (map[string]map[string]schema.MetricStatistics, error) {
	queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization?
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
	for i, res := range resBody.Results {
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}
		query := req.Queries[i]
		metric := query.Metric
		data := res[0]
		if data.Error != nil {
			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
			continue
		}

		metricdata, ok := stats[metric]
		if !ok {
			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
			stats[metric] = metricdata
		}

		if hasNaNStats(data.Avg, data.Min, data.Max) {
			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
			continue
		}

		metricdata[query.Hostname] = schema.MetricStatistics{
			Avg: float64(data.Avg),
			Min: float64(data.Min),
			Max: float64(data.Max),
		}
	}

	return stats, nil
}

// LoadScopedStats retrieves statistics for job metrics across multiple scopes.
// Used for the Job-View Statistics Table to display per-scope breakdowns.
//
// Returns statistics organized as: metric -> scope -> list of scoped statistics.
// Each scoped statistic includes hostname, hardware ID (if applicable), and min/avg/max values.
func (ccms *CCMetricStore) LoadScopedStats(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
) (schema.ScopedJobStats, error) {
	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	var errors []string
	scopedJobStats := make(schema.ScopedJobStats)

	for i, row := range resBody.Results {
		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]

		if _, ok := scopedJobStats[metric]; !ok {
			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
		}

		if _, ok := scopedJobStats[metric][scope]; !ok {
			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			sanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
				Hostname: query.Hostname,
				ID:       id,
				Data: &schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
			})
		}

		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
		if len(scopedJobStats[metric][scope]) == 0 {
			delete(scopedJobStats[metric], scope)
			if len(scopedJobStats[metric]) == 0 {
				delete(scopedJobStats, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return scopedJobStats, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return scopedJobStats, nil
}

// LoadNodeData retrieves current metric data for specified nodes in a cluster.
// Used for the Systems-View Node-Overview to display real-time node status.
//
// If nodes is nil, queries all metrics for all nodes in the cluster (bulk query).
// Returns data organized as: hostname -> metric -> list of JobMetric (with time series and stats).
func (ccms *CCMetricStore) LoadNodeData(
	cluster string,
	metrics, nodes []string,
	scopes []schema.MetricScope,
	from, to time.Time,
	ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
	req := APIQueryRequest{
		Cluster:   cluster,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	if nodes == nil {
		req.ForAllNodes = append(req.ForAllNodes, metrics...)
	} else {
		for _, node := range nodes {
			for _, metric := range metrics {
				req.Queries = append(req.Queries, APIQuery{
					Hostname:   node,
					Metric:     metric,
					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
				})
			}
		}
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]map[string][]*schema.JobMetric)
	for i, res := range resBody.Results {
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}

		var query APIQuery
		if resBody.Queries != nil {
			query = resBody.Queries[i]
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		qdata := res[0]
		if qdata.Error != nil {
			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
		}

		sanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)

		hostdata, ok := data[query.Hostname]
		if !ok {
			hostdata = make(map[string][]*schema.JobMetric)
			data[query.Hostname] = hostdata
		}

		mc := archive.GetMetricConfig(cluster, metric)
		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
			Unit:     mc.Unit,
			Timestep: mc.Timestep,
			Series: []schema.Series{
				{
					Hostname: query.Hostname,
					Data:     qdata.Data,
					Statistics: schema.MetricStatistics{
						Avg: float64(qdata.Avg),
						Min: float64(qdata.Min),
						Max: float64(qdata.Max),
					},
				},
			},
		})
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// LoadNodeListData retrieves paginated node metrics for the Systems-View Node-List.
//
// Supports filtering by subcluster and node name pattern. The nodeFilter performs
// substring matching on hostnames.
//
// Returns:
//   - Node data organized as: hostname -> JobData (metric -> scope -> series)
//   - Total node count (before pagination)
//   - HasNextPage flag indicating if more pages are available
//   - Error (may be partial error with some data returned)
func (ccms *CCMetricStore) LoadNodeListData(
	cluster, subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	from, to time.Time,
	ctx context.Context,
) (map[string]schema.JobData, error) {
	queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
	if err != nil {
		cclog.Errorf("Error while building node queries for Cluster %s, SubCluster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:   cluster,
		Queries:   queries,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]schema.JobData)

	// Add safety check for index out of range issues
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		// Safety check to prevent index out of range errors
		if i >= len(req.Queries) || i >= len(assignedScope) {
			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
				i, len(req.Queries), len(assignedScope))
			continue
		}

		var query APIQuery
		if resBody.Queries != nil {
			if i < len(resBody.Queries) {
				query = resBody.Queries[i]
			} else {
				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
					i, len(resBody.Queries))
				continue
			}
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
			continue
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = row[0].Resolution
		}

		// Init Nested Map Data Structures If Not Found
		hostData, ok := data[query.Hostname]
		if !ok {
			hostData = make(schema.JobData)
			data[query.Hostname] = hostData
		}

		metricData, ok := hostData[metric]
		if !ok {
			metricData = make(map[schema.MetricScope]*schema.JobMetric)
			data[query.Hostname][metric] = metricData
		}

		scopeData, ok := metricData[scope]
		if !ok {
			scopeData = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			data[query.Hostname][metric][scope] = scopeData
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			sanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopeData.Series = append(scopeData.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// HealthCheck queries the external cc-metric-store's health check endpoint.
// It sends a HealthCheckReq as the request body to /api/healthcheck and
// returns the per-node health check results.
func (ccms *CCMetricStore) HealthCheck(cluster string,
	nodes []string, metrics []string,
) (map[string]metricstore.HealthCheckResult, error) {
	req := metricstore.HealthCheckReq{
		Cluster:     cluster,
		Nodes:       nodes,
		MetricNames: metrics,
	}

	buf := &bytes.Buffer{}
	if err := json.NewEncoder(buf).Encode(req); err != nil {
		cclog.Errorf("Error while encoding health check request body: %s", err.Error())
		return nil, err
	}

	endpoint := fmt.Sprintf("%s/api/healthcheck", ccms.url)
	httpReq, err := http.NewRequest(http.MethodGet, endpoint, buf)
	if err != nil {
		cclog.Errorf("Error while building health check request: %s", err.Error())
		return nil, err
	}
	if ccms.jwt != "" {
		httpReq.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
	}

	res, err := ccms.client.Do(httpReq)
	if err != nil {
		cclog.Errorf("Error while performing health check request: %s", err.Error())
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("'%s': HTTP Status: %s", endpoint, res.Status)
	}

	var results map[string]metricstore.HealthCheckResult
	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&results); err != nil {
		cclog.Errorf("Error while decoding health check response: %s", err.Error())
		return nil, err
	}

	return results, nil
}

// sanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
// Regular float64 values cannot be JSONed when NaN.
func sanitizeStats(avg, min, max *schema.Float) {
	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
		*avg = schema.Float(0)
		*min = schema.Float(0)
		*max = schema.Float(0)
	}
}

// hasNaNStats returns true if any of the statistics contain NaN values.
func hasNaNStats(avg, min, max schema.Float) bool {
	return avg.IsNaN() || min.IsNaN() || max.IsNaN()
}


Now let me read the relevant parts of the metricstore query.go file to understand the buildQueries and buildNodeQueries functions, as well as the sanitizeStats and other helpers.


Now let me read the key sections of the metricstore query.go file.


   280→		}
   281→
   282→		// Skip if metric is removed for subcluster
   283→		if len(mc.SubClusters) != 0 {
   284→			isRemoved := false
   285→			for _, scConfig := range mc.SubClusters {
   286→				if scConfig.Name == job.SubCluster && scConfig.Remove {
   287→					isRemoved = true
   288→					break
   289→				}
   290→			}
   291→			if isRemoved {
   292→				continue
   293→			}
   294→		}
   295→
   296→		// Avoid duplicates using map for O(1) lookup
   297→		handledScopes := make(map[schema.MetricScope]bool, 3)
   298→
   299→		for _, requestedScope := range scopes {
   300→			nativeScope := mc.Scope
   301→			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
   302→				continue
   303→			}
   304→
   305→			scope := nativeScope.Max(requestedScope)
   306→			if handledScopes[scope] {
   307→				continue
   308→			}
   309→			handledScopes[scope] = true
   310→
   311→			for _, host := range job.Resources {
   312→				hwthreads := host.HWThreads
   313→				if hwthreads == nil {
   314→					hwthreads = topology.Node
   315→				}
   316→
   317→				// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
   318→				if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
   319→					if scope != schema.MetricScopeAccelerator {
   320→						// Skip all other catched cases
   321→						continue
   322→					}
   323→
   324→					queries = append(queries, APIQuery{
   325→						Metric:     metric,
   326→						Hostname:   host.Hostname,
   327→						Aggregate:  false,
   328→						Type:       &acceleratorString,
   329→						TypeIds:    host.Accelerators,
   330→						Resolution: resolution,
   331→					})
   332→					assignedScope = append(assignedScope, schema.MetricScopeAccelerator)
   333→					continue
   334→				}
   335→
   336→				// Accelerator -> Node
   337→				if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
   338→					if len(host.Accelerators) == 0 {
   339→						continue
   340→					}
   341→
   342→					queries = append(queries, APIQuery{
   343→						Metric:     metric,
   344→						Hostname:   host.Hostname,
   345→						Aggregate:  true,
   346→						Type:       &acceleratorString,
   347→						TypeIds:    host.Accelerators,
   348→						Resolution: resolution,
   349→					})
   350→					assignedScope = append(assignedScope, scope)
   351→					continue
   352→				}
   353→
   354→				// HWThread -> HWThread
   355→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
   356→					queries = append(queries, APIQuery{
   357→						Metric:     metric,
   358→						Hostname:   host.Hostname,
   359→						Aggregate:  false,
   360→						Type:       &hwthreadString,
   361→						TypeIds:    intToStringSlice(hwthreads),
   362→						Resolution: resolution,
   363→					})
   364→					assignedScope = append(assignedScope, scope)
   365→					continue
   366→				}
   367→
   368→				// HWThread -> Core
   369→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
   370→					cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   371→					for _, core := range cores {
   372→						queries = append(queries, APIQuery{
   373→							Metric:     metric,
   374→							Hostname:   host.Hostname,
   375→							Aggregate:  true,
   376→							Type:       &hwthreadString,
   377→							TypeIds:    intToStringSlice(topology.Core[core]),
   378→							Resolution: resolution,
   379→						})
   380→						assignedScope = append(assignedScope, scope)
   381→					}
   382→					continue
   383→				}
   384→
   385→				// HWThread -> Socket
   386→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
   387→					sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   388→					for _, socket := range sockets {
   389→						queries = append(queries, APIQuery{
   390→							Metric:     metric,
   391→							Hostname:   host.Hostname,
   392→							Aggregate:  true,
   393→							Type:       &hwthreadString,
   394→							TypeIds:    intToStringSlice(topology.Socket[socket]),
   395→							Resolution: resolution,
   396→						})
   397→						assignedScope = append(assignedScope, scope)
   398→					}
   399→					continue
   400→				}
   401→
   402→				// HWThread -> Node
   403→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
   404→					queries = append(queries, APIQuery{
   405→						Metric:     metric,
   406→						Hostname:   host.Hostname,
   407→						Aggregate:  true,
   408→						Type:       &hwthreadString,
   409→						TypeIds:    intToStringSlice(hwthreads),
   410→						Resolution: resolution,
   411→					})
   412→					assignedScope = append(assignedScope, scope)
   413→					continue
   414→				}
   415→
   416→				// Core -> Core
   417→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
   418→					cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   419→					queries = append(queries, APIQuery{
   420→						Metric:     metric,
   421→						Hostname:   host.Hostname,
   422→						Aggregate:  false,
   423→						Type:       &coreString,
   424→						TypeIds:    intToStringSlice(cores),
   425→						Resolution: resolution,
   426→					})
   427→					assignedScope = append(assignedScope, scope)
   428→					continue
   429→				}
   430→
   431→				// Core -> Socket
   432→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
   433→					sockets, _ := topology.GetSocketsFromCores(hwthreads)
   434→					for _, socket := range sockets {
   435→						queries = append(queries, APIQuery{
   436→							Metric:     metric,
   437→							Hostname:   host.Hostname,
   438→							Aggregate:  true,
   439→							Type:       &coreString,
   440→							TypeIds:    intToStringSlice(topology.Socket[socket]),
   441→							Resolution: resolution,
   442→						})
   443→						assignedScope = append(assignedScope, scope)
   444→					}
   445→					continue
   446→				}
   447→
   448→				// Core -> Node
   449→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
   450→					cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   451→					queries = append(queries, APIQuery{
   452→						Metric:     metric,
   453→						Hostname:   host.Hostname,
   454→						Aggregate:  true,
   455→						Type:       &coreString,
   456→						TypeIds:    intToStringSlice(cores),
   457→						Resolution: resolution,
   458→					})
   459→					assignedScope = append(assignedScope, scope)
   460→					continue
   461→				}
   462→
   463→				// MemoryDomain -> MemoryDomain
   464→				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
   465→					sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   466→					queries = append(queries, APIQuery{
   467→						Metric:     metric,
   468→						Hostname:   host.Hostname,
   469→						Aggregate:  false,
   470→						Type:       &memoryDomainString,
   471→						TypeIds:    intToStringSlice(sockets),
   472→						Resolution: resolution,
   473→					})
   474→					assignedScope = append(assignedScope, scope)
   475→					continue
   476→				}
   477→
   478→				// MemoryDomain -> Node
   479→				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
   480→					sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   481→					queries = append(queries, APIQuery{
   482→						Metric:     metric,
   483→						Hostname:   host.Hostname,
   484→						Aggregate:  true,
   485→						Type:       &memoryDomainString,
   486→						TypeIds:    intToStringSlice(sockets),
   487→						Resolution: resolution,
   488→					})
   489→					assignedScope = append(assignedScope, scope)
   490→					continue
   491→				}
   492→
   493→				// Socket -> Socket
   494→				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
   495→					sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   496→					queries = append(queries, APIQuery{
   497→						Metric:     metric,
   498→						Hostname:   host.Hostname,
   499→						Aggregate:  false,
   500→						Type:       &socketString,
   501→						TypeIds:    intToStringSlice(sockets),
   502→						Resolution: resolution,
   503→					})
   504→					assignedScope = append(assignedScope, scope)
   505→					continue
   506→				}
   507→
   508→				// Socket -> Node
   509→				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
   510→					sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   511→					queries = append(queries, APIQuery{
   512→						Metric:     metric,
   513→						Hostname:   host.Hostname,
   514→						Aggregate:  true,
   515→						Type:       &socketString,
   516→						TypeIds:    intToStringSlice(sockets),
   517→						Resolution: resolution,
   518→					})
   519→					assignedScope = append(assignedScope, scope)
   520→					continue
   521→				}
   522→
   523→				// Node -> Node
   524→				if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
   525→					queries = append(queries, APIQuery{
   526→						Metric:     metric,
   527→						Hostname:   host.Hostname,
   528→						Resolution: resolution,
   529→					})
   530→					assignedScope = append(assignedScope, scope)
   531→					continue
   532→				}
   533→
   534→				return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   535→			}
   536→		}
   537→	}
   538→
   539→	return queries, assignedScope, nil
   540→}
   541→
   542→// LoadStats loads only metric statistics (avg/min/max) for a job at node scope.
   543→//
   544→// This is an optimized version of LoadData that fetches only statistics without
   545→// time-series data, reducing bandwidth and memory usage. Always queries at node scope.
   546→//
   547→// Parameters:
   548→//   - job: Job metadata
   549→//   - metrics: List of metric names
   550→//   - ctx: Context (currently unused)
   551→//
   552→// Returns:
   553→//   - Map of metric → hostname → statistics
   554→//   - Error on query building or fetching failure
   555→func (ccms *InternalMetricStore) LoadStats(
   556→	job *schema.Job,
   557→	metrics []string,
   558→	ctx context.Context,
   559→) (map[string]map[string]schema.MetricStatistics, error) {
   560→	// TODO(#166): Add scope parameter for analysis view accelerator normalization
   561→	queries, _, err := buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0)
   562→	if err != nil {
   563→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
   564→		return nil, err
   565→	}
   566→
   567→	req := APIQueryRequest{
   568→		Cluster:   job.Cluster,
   569→		From:      job.StartTime,
   570→		To:        job.StartTime + int64(job.Duration),
   571→		Queries:   queries,
   572→		WithStats: true,
   573→		WithData:  false,
   574→	}
   575→
   576→	resBody, err := FetchData(req)
   577→	if err != nil {
   578→		cclog.Errorf("Error while fetching data : %s", err.Error())
   579→		return nil, err

		}

		// Skip if metric is removed for subcluster
		if len(mc.SubClusters) != 0 {
			isRemoved := false
			for _, scConfig := range mc.SubClusters {
				if scConfig.Name == job.SubCluster && scConfig.Remove {
					isRemoved = true
					break
				}
			}
			if isRemoved {
				continue
			}
		}

		// Avoid duplicates using map for O(1) lookup
		handledScopes := make(map[schema.MetricScope]bool, 3)

		for _, requestedScope := range scopes {
			nativeScope := mc.Scope
			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
				continue
			}

			scope := nativeScope.Max(requestedScope)
			if handledScopes[scope] {
				continue
			}
			handledScopes[scope] = true

			for _, host := range job.Resources {
				hwthreads := host.HWThreads
				if hwthreads == nil {
					hwthreads = topology.Node
				}

				// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
				if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
					if scope != schema.MetricScopeAccelerator {
						// Skip all other catched cases
						continue
					}

					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   host.Hostname,
						Aggregate:  false,
						Type:       &acceleratorString,
						TypeIds:    host.Accelerators,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, schema.MetricScopeAccelerator)
					continue
				}

				// Accelerator -> Node
				if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
					if len(host.Accelerators) == 0 {
						continue
					}

					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   host.Hostname,
						Aggregate:  true,
						Type:       &acceleratorString,
						TypeIds:    host.Accelerators,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// HWThread -> HWThread
				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   host.Hostname,
						Aggregate:  false,
						Type:       &hwthreadString,
						TypeIds:    intToStringSlice(hwthreads),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// HWThread -> Core
				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
					cores, _ := topology.GetCoresFromHWThreads(hwthreads)
					for _, core := range cores {
						queries = append(queries, APIQuery{
							Metric:     metric,
							Hostname:   host.Hostname,
							Aggregate:  true,
							Type:       &hwthreadString,
							TypeIds:    intToStringSlice(topology.Core[core]),
							Resolution: resolution,
						})
						assignedScope = append(assignedScope, scope)
					}
					continue
				}

				// HWThread -> Socket
				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
					sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
					for _, socket := range sockets {
						queries = append(queries, APIQuery{
							Metric:     metric,
							Hostname:   host.Hostname,
							Aggregate:  true,
							Type:       &hwthreadString,
							TypeIds:    intToStringSlice(topology.Socket[socket]),
							Resolution: resolution,
						})
						assignedScope = append(assignedScope, scope)
					}
					continue
				}

				// HWThread -> Node
				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   host.Hostname,
						Aggregate:  true,
						Type:       &hwthreadString,
						TypeIds:    intToStringSlice(hwthreads),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// Core -> Core
				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
					cores, _ := topology.GetCoresFromHWThreads(hwthreads)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   host.Hostname,
						Aggregate:  false,
						Type:       &coreString,
						TypeIds:    intToStringSlice(cores),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// Core -> Socket
				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
					sockets, _ := topology.GetSocketsFromCores(hwthreads)
					for _, socket := range sockets {
						queries = append(queries, APIQuery{
							Metric:     metric,
							Hostname:   host.Hostname,
							Aggregate:  true,
							Type:       &coreString,
							TypeIds:    intToStringSlice(topology.Socket[socket]),
							Resolution: resolution,
						})
						assignedScope = append(assignedScope, scope)
					}
					continue
				}

				// Core -> Node
				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
					cores, _ := topology.GetCoresFromHWThreads(hwthreads)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   host.Hostname,
						Aggregate:  true,
						Type:       &coreString,
						TypeIds:    intToStringSlice(cores),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// MemoryDomain -> MemoryDomain
				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
					sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   host.Hostname,
						Aggregate:  false,
						Type:       &memoryDomainString,
						TypeIds:    intToStringSlice(sockets),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// MemoryDomain -> Node
				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
					sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   host.Hostname,
						Aggregate:  true,
						Type:       &memoryDomainString,
						TypeIds:    intToStringSlice(sockets),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// Socket -> Socket
				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
					sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   host.Hostname,
						Aggregate:  false,
						Type:       &socketString,
						TypeIds:    intToStringSlice(sockets),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// Socket -> Node
				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
					sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   host.Hostname,
						Aggregate:  true,
						Type:       &socketString,
						TypeIds:    intToStringSlice(sockets),
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				// Node -> Node
				if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
					queries = append(queries, APIQuery{
						Metric:     metric,
						Hostname:   host.Hostname,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, scope)
					continue
				}

				return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
			}
		}
	}

	return queries, assignedScope, nil
}

// LoadStats loads only metric statistics (avg/min/max) for a job at node scope.
//
// This is an optimized version of LoadData that fetches only statistics without
// time-series data, reducing bandwidth and memory usage. Always queries at node scope.
//
// Parameters:
//   - job: Job metadata
//   - metrics: List of metric names
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of metric → hostname → statistics
//   - Error on query building or fetching failure
func (ccms *InternalMetricStore) LoadStats(
	job *schema.Job,
	metrics []string,
	ctx context.Context,
) (map[string]map[string]schema.MetricStatistics, error) {
	// TODO(#166): Add scope parameter for analysis view accelerator normalization
	queries, _, err := buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements high-level query functions for loading job metric data
     7→// with automatic scope transformation and aggregation.
     8→//
     9→// Key Concepts:
    10→//
    11→// Metric Scopes: Metrics are collected at different granularities (native scope):
    12→//   - HWThread: Per hardware thread
    13→//   - Core: Per CPU core
    14→//   - Socket: Per CPU socket
    15→//   - MemoryDomain: Per memory domain (NUMA)
    16→//   - Accelerator: Per GPU/accelerator
    17→//   - Node: Per compute node
    18→//
    19→// Scope Transformation: The buildQueries functions transform between native scope
    20→// and requested scope by:
    21→//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
    22→//   - Rejecting requests for finer granularity than available
    23→//   - Handling special cases (e.g., Accelerator metrics)
    24→//
    25→// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
    26→// based on cluster topology and job resources.
    27→package metricstore
    28→
    29→import (
    30→	"context"
    31→	"fmt"
    32→	"strconv"
    33→	"strings"
    34→	"time"
    35→
    36→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    37→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    38→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    39→)
    40→
    41→type InternalMetricStore struct{}
    42→
    43→var MetricStoreHandle *InternalMetricStore
    44→
    45→// HealthCheck delegates to the internal MemoryStore's HealthCheck.
    46→func (ccms *InternalMetricStore) HealthCheck(cluster string,
    47→	nodes []string, metrics []string,
    48→) (map[string]HealthCheckResult, error) {
    49→	return GetMemoryStore().HealthCheck(cluster, nodes, metrics)
    50→}
    51→
    52→// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
    53→// When set to a non-nil function, LoadData will call this function instead of the default implementation.
    54→var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)
    55→
    56→// LoadData loads metric data for a specific job with automatic scope transformation.
    57→//
    58→// This is the primary function for retrieving job metric data. It handles:
    59→//   - Building queries with scope transformation via buildQueries
    60→//   - Fetching data from the metric store
    61→//   - Organizing results by metric and scope
    62→//   - Converting NaN statistics to 0 for JSON compatibility
    63→//   - Partial error handling (returns data for successful queries even if some fail)
    64→//
    65→// Parameters:
    66→//   - job: Job metadata including cluster, resources, and time range
    67→//   - metrics: List of metric names to load
    68→//   - scopes: Requested metric scopes (will be transformed to match native scopes)
    69→//   - ctx: Context for cancellation (currently unused but reserved for future use)
    70→//   - resolution: Data resolution in seconds (0 for native resolution)
    71→//
    72→// Returns:
    73→//   - JobData: Map of metric → scope → JobMetric with time-series data and statistics
    74→//   - Error: Returns error if query building or fetching fails, or partial error listing failed hosts
    75→//
    76→// Example:
    77→//
    78→//	jobData, err := LoadData(job, []string{"cpu_load", "mem_used"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 60)
    79→func (ccms *InternalMetricStore) LoadData(
    80→	job *schema.Job,
    81→	metrics []string,
    82→	scopes []schema.MetricScope,
    83→	ctx context.Context,
    84→	resolution int,
    85→) (schema.JobData, error) {
    86→	if TestLoadDataCallback != nil {
    87→		return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
    88→	}
    89→
    90→	queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
    91→	if err != nil {
    92→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
    93→		return nil, err
    94→	}
    95→
    96→	// Verify assignment is correct - log any inconsistencies for debugging
    97→	if len(queries) != len(assignedScope) {
    98→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
    99→			len(queries), len(assignedScope))
   100→	}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements high-level query functions for loading job metric data
// with automatic scope transformation and aggregation.
//
// Key Concepts:
//
// Metric Scopes: Metrics are collected at different granularities (native scope):
//   - HWThread: Per hardware thread
//   - Core: Per CPU core
//   - Socket: Per CPU socket
//   - MemoryDomain: Per memory domain (NUMA)
//   - Accelerator: Per GPU/accelerator
//   - Node: Per compute node
//
// Scope Transformation: The buildQueries functions transform between native scope
// and requested scope by:
//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
//   - Rejecting requests for finer granularity than available
//   - Handling special cases (e.g., Accelerator metrics)
//
// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
// based on cluster topology and job resources.
package metricstore

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

type InternalMetricStore struct{}

var MetricStoreHandle *InternalMetricStore

// HealthCheck delegates to the internal MemoryStore's HealthCheck.
func (ccms *InternalMetricStore) HealthCheck(cluster string,
	nodes []string, metrics []string,
) (map[string]HealthCheckResult, error) {
	return GetMemoryStore().HealthCheck(cluster, nodes, metrics)
}

// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
// When set to a non-nil function, LoadData will call this function instead of the default implementation.
var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)

// LoadData loads metric data for a specific job with automatic scope transformation.
//
// This is the primary function for retrieving job metric data. It handles:
//   - Building queries with scope transformation via buildQueries
//   - Fetching data from the metric store
//   - Organizing results by metric and scope
//   - Converting NaN statistics to 0 for JSON compatibility
//   - Partial error handling (returns data for successful queries even if some fail)
//
// Parameters:
//   - job: Job metadata including cluster, resources, and time range
//   - metrics: List of metric names to load
//   - scopes: Requested metric scopes (will be transformed to match native scopes)
//   - ctx: Context for cancellation (currently unused but reserved for future use)
//   - resolution: Data resolution in seconds (0 for native resolution)
//
// Returns:
//   - JobData: Map of metric → scope → JobMetric with time-series data and statistics
//   - Error: Returns error if query building or fetching fails, or partial error listing failed hosts
//
// Example:
//
//	jobData, err := LoadData(job, []string{"cpu_load", "mem_used"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 60)
func (ccms *InternalMetricStore) LoadData(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
	resolution int,
) (schema.JobData, error) {
	if TestLoadDataCallback != nil {
		return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
	}

	queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
			len(queries), len(assignedScope))
	}