   200→			})
   201→		}
   202→
   203→		// So that one can later check len(jobData):
   204→		if len(jobMetric.Series) == 0 {
   205→			delete(jobData[metric], scope)
   206→			if len(jobData[metric]) == 0 {
   207→				delete(jobData, metric)
   208→			}
   209→		}
   210→	}
   211→
   212→	if len(errors) != 0 {
   213→		/* Returns list for "partial errors" */
   214→		return jobData, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   215→	}
   216→	return jobData, nil
   217→}
   218→
   219→// Pre-converted scope strings avoid repeated string(MetricScope) allocations during
   220→// query construction. These are used in APIQuery.Type field throughout buildQueries
   221→// and buildNodeQueries functions. Converting once at package initialization improves
   222→// performance for high-volume query building.
   223→var (
   224→	hwthreadString     = string(schema.MetricScopeHWThread)
   225→	coreString         = string(schema.MetricScopeCore)
   226→	memoryDomainString = string(schema.MetricScopeMemoryDomain)
   227→	socketString       = string(schema.MetricScopeSocket)
   228→	acceleratorString  = string(schema.MetricScopeAccelerator)
   229→)
   230→
   231→// buildQueries constructs APIQuery structures with automatic scope transformation for a job.
   232→//
   233→// This function implements the core scope transformation logic, handling all combinations of
   234→// native metric scopes and requested scopes. It uses the cluster topology to determine which
   235→// hardware IDs to include in each query.
   236→//
   237→// Scope Transformation Rules:
   238→//   - If native scope >= requested scope: Aggregates data (Aggregate=true in APIQuery)
   239→//   - If native scope < requested scope: Returns error (cannot increase granularity)
   240→//   - Special handling for Accelerator scope (independent of CPU hierarchy)
   241→//
   242→// The function generates one or more APIQuery per (metric, scope, host) combination:
   243→//   - For non-aggregated queries: One query with all relevant IDs
   244→//   - For aggregated queries: May generate multiple queries (e.g., one per socket/core)
   245→//
   246→// Parameters:
   247→//   - job: Job metadata including cluster, subcluster, and resource allocation
   248→//   - metrics: List of metrics to query
   249→//   - scopes: Requested scopes for each metric
   250→//   - resolution: Data resolution in seconds
   251→//
   252→// Returns:
   253→//   - []APIQuery: List of queries to execute
   254→//   - []schema.MetricScope: Assigned scope for each query (after transformation)
   255→//   - error: Returns error if topology lookup fails or unhandled scope combination encountered
   256→func buildQueries(
   257→	job *schema.Job,
   258→	metrics []string,
   259→	scopes []schema.MetricScope,
   260→	resolution int64,
   261→) ([]APIQuery, []schema.MetricScope, error) {
   262→	if len(job.Resources) == 0 {
   263→		return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > no resources allocated for job %d", job.JobID)
   264→	}
   265→
   266→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
   267→	assignedScope := []schema.MetricScope{}
   268→
   269→	subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster)
   270→	if scerr != nil {
   271→		return nil, nil, scerr
   272→	}
   273→	topology := subcluster.Topology
   274→
   275→	for _, metric := range metrics {
   276→		mc := archive.GetMetricConfig(job.Cluster, metric)
   277→		if mc == nil {
   278→			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
   279→			continue
   280→		}
   281→
   282→		// Skip if metric is removed for subcluster
   283→		if len(mc.SubClusters) != 0 {
   284→			isRemoved := false
   285→			for _, scConfig := range mc.SubClusters {
   286→				if scConfig.Name == job.SubCluster && scConfig.Remove {
   287→					isRemoved = true
   288→					break
   289→				}
   290→			}
   291→			if isRemoved {
   292→				continue
   293→			}
   294→		}
   295→
   296→		// Avoid duplicates using map for O(1) lookup
   297→		handledScopes := make(map[schema.MetricScope]bool, 3)
   298→
   299→		for _, requestedScope := range scopes {
   300→			nativeScope := mc.Scope
   301→			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
   302→				continue
   303→			}
   304→
   305→			scope := nativeScope.Max(requestedScope)
   306→			if handledScopes[scope] {
   307→				continue
   308→			}
   309→			handledScopes[scope] = true
   310→
   311→			for _, host := range job.Resources {
   312→				hwthreads := host.HWThreads
   313→				if hwthreads == nil {
   314→					hwthreads = topology.Node
   315→				}
   316→
   317→				// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
   318→				if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
   319→					if scope != schema.MetricScopeAccelerator {
   320→						// Skip all other catched cases
   321→						continue
   322→					}
   323→
   324→					queries = append(queries, APIQuery{
   325→						Metric:     metric,
   326→						Hostname:   host.Hostname,
   327→						Aggregate:  false,
   328→						Type:       &acceleratorString,
   329→						TypeIds:    host.Accelerators,
   330→						Resolution: resolution,
   331→					})
   332→					assignedScope = append(assignedScope, schema.MetricScopeAccelerator)
   333→					continue
   334→				}
   335→
   336→				// Accelerator -> Node
   337→				if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
   338→					if len(host.Accelerators) == 0 {
   339→						continue
   340→					}
   341→
   342→					queries = append(queries, APIQuery{
   343→						Metric:     metric,
   344→						Hostname:   host.Hostname,
   345→						Aggregate:  true,
   346→						Type:       &acceleratorString,
   347→						TypeIds:    host.Accelerators,
   348→						Resolution: resolution,
   349→					})
   350→					assignedScope = append(assignedScope, scope)
   351→					continue
   352→				}
   353→
   354→				// HWThread -> HWThread
   355→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
   356→					queries = append(queries, APIQuery{
   357→						Metric:     metric,
   358→						Hostname:   host.Hostname,
   359→						Aggregate:  false,
   360→						Type:       &hwthreadString,
   361→						TypeIds:    intToStringSlice(hwthreads),
   362→						Resolution: resolution,
   363→					})
   364→					assignedScope = append(assignedScope, scope)
   365→					continue
   366→				}
   367→
   368→				// HWThread -> Core
   369→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
   370→					cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   371→					for _, core := range cores {
   372→						queries = append(queries, APIQuery{
   373→							Metric:     metric,
   374→							Hostname:   host.Hostname,
   375→							Aggregate:  true,
   376→							Type:       &hwthreadString,
   377→							TypeIds:    intToStringSlice(topology.Core[core]),
   378→							Resolution: resolution,
   379→						})
   380→						assignedScope = append(assignedScope, scope)
   381→					}
   382→					continue
   383→				}
   384→
   385→				// HWThread -> Socket
   386→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
   387→					sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   388→					for _, socket := range sockets {
   389→						queries = append(queries, APIQuery{
   390→							Metric:     metric,
   391→							Hostname:   host.Hostname,
   392→							Aggregate:  true,
   393→							Type:       &hwthreadString,
   394→							TypeIds:    intToStringSlice(topology.Socket[socket]),
   395→							Resolution: resolution,
   396→						})
   397→						assignedScope = append(assignedScope, scope)
   398→					}
   399→					continue
   400→				}
   401→
   402→				// HWThread -> Node
   403→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
   404→					queries = append(queries, APIQuery{
   405→						Metric:     metric,
   406→						Hostname:   host.Hostname,
   407→						Aggregate:  true,
   408→						Type:       &hwthreadString,
   409→						TypeIds:    intToStringSlice(hwthreads),
   410→						Resolution: resolution,
   411→					})
   412→					assignedScope = append(assignedScope, scope)
   413→					continue
   414→				}
   415→
   416→				// Core -> Core
   417→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
   418→					cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   419→					queries = append(queries, APIQuery{
   420→						Metric:     metric,
   421→						Hostname:   host.Hostname,
   422→						Aggregate:  false,
   423→						Type:       &coreString,
   424→						TypeIds:    intToStringSlice(cores),
   425→						Resolution: resolution,
   426→					})
   427→					assignedScope = append(assignedScope, scope)
   428→					continue
   429→				}
   430→
   431→				// Core -> Socket
   432→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
   433→					sockets, _ := topology.GetSocketsFromCores(hwthreads)
   434→					for _, socket := range sockets {
   435→						queries = append(queries, APIQuery{
   436→							Metric:     metric,
   437→							Hostname:   host.Hostname,
   438→							Aggregate:  true,
   439→							Type:       &coreString,
   440→							TypeIds:    intToStringSlice(topology.Socket[socket]),
   441→							Resolution: resolution,
   442→						})
   443→						assignedScope = append(assignedScope, scope)
   444→					}
   445→					continue
   446→				}
   447→
   448→				// Core -> Node
   449→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
   450→					cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   451→					queries = append(queries, APIQuery{
   452→						Metric:     metric,
   453→						Hostname:   host.Hostname,
   454→						Aggregate:  true,
   455→						Type:       &coreString,
   456→						TypeIds:    intToStringSlice(cores),
   457→						Resolution: resolution,
   458→					})
   459→					assignedScope = append(assignedScope, scope)
   460→					continue
   461→				}
   462→
   463→				// MemoryDomain -> MemoryDomain
   464→				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
   465→					sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   466→					queries = append(queries, APIQuery{
   467→						Metric:     metric,
   468→						Hostname:   host.Hostname,
   469→						Aggregate:  false,
   470→						Type:       &memoryDomainString,
   471→						TypeIds:    intToStringSlice(sockets),
   472→						Resolution: resolution,
   473→					})
   474→					assignedScope = append(assignedScope, scope)
   475→					continue
   476→				}
   477→
   478→				// MemoryDomain -> Node
   479→				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
   480→					sockets, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   481→					queries = append(queries, APIQuery{
   482→						Metric:     metric,
   483→						Hostname:   host.Hostname,
   484→						Aggregate:  true,
   485→						Type:       &memoryDomainString,
   486→						TypeIds:    intToStringSlice(sockets),
   487→						Resolution: resolution,
   488→					})
   489→					assignedScope = append(assignedScope, scope)
   490→					continue
   491→				}
   492→
   493→				// Socket -> Socket
   494→				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
   495→					sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   496→					queries = append(queries, APIQuery{
   497→						Metric:     metric,
   498→						Hostname:   host.Hostname,
   499→						Aggregate:  false,
   500→						Type:       &socketString,
   501→						TypeIds:    intToStringSlice(sockets),
   502→						Resolution: resolution,
   503→					})
   504→					assignedScope = append(assignedScope, scope)
   505→					continue
   506→				}
   507→
   508→				// Socket -> Node
   509→				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
   510→					sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   511→					queries = append(queries, APIQuery{
   512→						Metric:     metric,
   513→						Hostname:   host.Hostname,
   514→						Aggregate:  true,
   515→						Type:       &socketString,
   516→						TypeIds:    intToStringSlice(sockets),
   517→						Resolution: resolution,
   518→					})
   519→					assignedScope = append(assignedScope, scope)
   520→					continue
   521→				}
   522→
   523→				// Node -> Node
   524→				if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
   525→					queries = append(queries, APIQuery{
   526→						Metric:     metric,
   527→						Hostname:   host.Hostname,
   528→						Resolution: resolution,
   529→					})
   530→					assignedScope = append(assignedScope, scope)
   531→					continue
   532→				}
   533→
   534→				return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   535→			}
   536→		}
   537→	}
   538→
   539→	return queries, assignedScope, nil
   540→}
   541→
   542→// LoadStats loads only metric statistics (avg/min/max) for a job at node scope.
   543→//
   544→// This is an optimized version of LoadData that fetches only statistics without
   545→// time-series data, reducing bandwidth and memory usage. Always queries at node scope.
   546→//
   547→// Parameters:
   548→//   - job: Job metadata
   549→//   - metrics: List of metric names
   550→//   - ctx: Context (currently unused)
   551→//
   552→// Returns:
   553→//   - Map of metric → hostname → statistics
   554→//   - Error on query building or fetching failure
   555→func (ccms *InternalMetricStore) LoadStats(
   556→	job *schema.Job,
   557→	metrics []string,
   558→	ctx context.Context,
   559→) (map[string]map[string]schema.MetricStatistics, error) {
   560→	// TODO(#166): Add scope parameter for analysis view accelerator normalization
   561→	queries, _, err := buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0)
   562→	if err != nil {
   563→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
   564→		return nil, err
   565→	}
   566→
   567→	req := APIQueryRequest{
   568→		Cluster:   job.Cluster,
   569→		From:      job.StartTime,
   570→		To:        job.StartTime + int64(job.Duration),
   571→		Queries:   queries,
   572→		WithStats: true,
   573→		WithData:  false,
   574→	}
   575→
   576→	resBody, err := FetchData(req)
   577→	if err != nil {
   578→		cclog.Errorf("Error while fetching data : %s", err.Error())
   579→		return nil, err
   580→	}
   581→
   582→	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
   583→	for i, res := range resBody.Results {
   584→		if len(res) == 0 {
   585→			// No Data Found For Metric, Logged in FetchData to Warn
   586→			continue
   587→		}
   588→		query := req.Queries[i]
   589→		metric := query.Metric
   590→		data := res[0]
   591→		if data.Error != nil {
   592→			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
   593→			continue
   594→		}
   595→
   596→		metricdata, ok := stats[metric]
   597→		if !ok {
   598→			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
   599→			stats[metric] = metricdata
   600→		}
   601→
   602→		if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() {
   603→			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
   604→			continue
   605→		}
   606→
   607→		metricdata[query.Hostname] = schema.MetricStatistics{
   608→			Avg: float64(data.Avg),
   609→			Min: float64(data.Min),
   610→			Max: float64(data.Max),
   611→		}
   612→	}
   613→
   614→	return stats, nil
   615→}
   616→
   617→// LoadScopedStats loads metric statistics for a job with scope-aware grouping.
   618→//
   619→// Similar to LoadStats but supports multiple scopes and returns statistics grouped
   620→// by scope with hardware IDs (e.g., per-core, per-socket statistics).
   621→//
   622→// Parameters:
   623→//   - job: Job metadata
   624→//   - metrics: List of metric names
   625→//   - scopes: Requested metric scopes
   626→//   - ctx: Context (currently unused)
   627→//
   628→// Returns:
   629→//   - ScopedJobStats: Map of metric → scope → []ScopedStats (with hostname and ID)
   630→//   - Error or partial error listing failed queries
   631→func (ccms *InternalMetricStore) LoadScopedStats(
   632→	job *schema.Job,
   633→	metrics []string,
   634→	scopes []schema.MetricScope,
   635→	ctx context.Context,
   636→) (schema.ScopedJobStats, error) {
   637→	queries, assignedScope, err := buildQueries(job, metrics, scopes, 0)
   638→	if err != nil {
   639→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   640→		return nil, err
   641→	}
   642→
   643→	req := APIQueryRequest{
   644→		Cluster:   job.Cluster,
   645→		From:      job.StartTime,
   646→		To:        job.StartTime + int64(job.Duration),
   647→		Queries:   queries,
   648→		WithStats: true,
   649→		WithData:  false,
   650→	}
   651→
   652→	resBody, err := FetchData(req)
   653→	if err != nil {
   654→		cclog.Errorf("Error while fetching data : %s", err.Error())
   655→		return nil, err
   656→	}
   657→
   658→	var errors []string
   659→	scopedJobStats := make(schema.ScopedJobStats)
   660→
   661→	for i, row := range resBody.Results {
   662→		if len(row) == 0 {
   663→			// No Data Found For Metric, Logged in FetchData to Warn
   664→			continue
   665→		}
   666→		query := req.Queries[i]
   667→		metric := query.Metric
   668→		scope := assignedScope[i]
   669→
   670→		if _, ok := scopedJobStats[metric]; !ok {
   671→			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
   672→		}
   673→
   674→		if _, ok := scopedJobStats[metric][scope]; !ok {
   675→			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
   676→		}
   677→
   678→		for ndx, res := range row {
   679→			if res.Error != nil {
   680→				/* Build list for "partial errors", if any */
   681→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   682→				continue
   683→			}
   684→
   685→			id := (*string)(nil)
   686→			if query.Type != nil {
   687→				// Check if ndx is within the bounds of TypeIds slice
   688→				if ndx < len(query.TypeIds) {
   689→					id = new(string)
   690→					*id = query.TypeIds[ndx]
   691→				} else {
   692→					// Log the error but continue processing
   693→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   694→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   695→				}
   696→			}
   697→
   698→			sanitizeStats(&res)
   699→
   700→			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
   701→				Hostname: query.Hostname,
   702→				ID:       id,
   703→				Data: &schema.MetricStatistics{
   704→					Avg: float64(res.Avg),
   705→					Min: float64(res.Min),
   706→					Max: float64(res.Max),
   707→				},
   708→			})
   709→		}
   710→
   711→		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
   712→		if len(scopedJobStats[metric][scope]) == 0 {
   713→			delete(scopedJobStats[metric], scope)
   714→			if len(scopedJobStats[metric]) == 0 {
   715→				delete(scopedJobStats, metric)
   716→			}
   717→		}
   718→	}
   719→
   720→	if len(errors) != 0 {
   721→		/* Returns list for "partial errors" */
   722→		return scopedJobStats, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   723→	}
   724→	return scopedJobStats, nil
   725→}
   726→
   727→// LoadNodeData loads metric data for specific nodes in a cluster over a time range.
   728→//
   729→// Unlike LoadData which operates on job resources, this function queries arbitrary nodes
   730→// directly. Useful for system monitoring and node status views.
   731→//
   732→// Parameters:
   733→//   - cluster: Cluster name
   734→//   - metrics: List of metric names
   735→//   - nodes: List of node hostnames (nil = all nodes in cluster via ForAllNodes)
   736→//   - scopes: Requested metric scopes (currently unused - always node scope)
   737→//   - from, to: Time range
   738→//   - ctx: Context (currently unused)
   739→//
   740→// Returns:
   741→//   - Map of hostname → metric → []JobMetric
   742→//   - Error or partial error listing failed queries
   743→func (ccms *InternalMetricStore) LoadNodeData(
   744→	cluster string,
   745→	metrics, nodes []string,
   746→	scopes []schema.MetricScope,
   747→	from, to time.Time,
   748→	ctx context.Context,
   749→) (map[string]map[string][]*schema.JobMetric, error) {
   750→	req := APIQueryRequest{
   751→		Cluster:   cluster,
   752→		From:      from.Unix(),
   753→		To:        to.Unix(),
   754→		WithStats: true,
   755→		WithData:  true,
   756→	}
   757→
   758→	if nodes == nil {
   759→		req.ForAllNodes = append(req.ForAllNodes, metrics...)
   760→	} else {
   761→		for _, node := range nodes {
   762→			for _, metric := range metrics {
   763→				req.Queries = append(req.Queries, APIQuery{
   764→					Hostname:   node,
   765→					Metric:     metric,
   766→					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
   767→				})
   768→			}
   769→		}
   770→	}
   771→
   772→	resBody, err := FetchData(req)
   773→	if err != nil {
   774→		cclog.Errorf("Error while fetching data : %s", err.Error())
   775→		return nil, err
   776→	}
   777→
   778→	var errors []string
   779→	data := make(map[string]map[string][]*schema.JobMetric)
   780→	for i, res := range resBody.Results {
   781→		if len(res) == 0 {
   782→			// No Data Found For Metric, Logged in FetchData to Warn
   783→			continue
   784→		}
   785→
   786→		var query APIQuery
   787→		if resBody.Queries != nil {
   788→			query = resBody.Queries[i]
   789→		} else {
   790→			query = req.Queries[i]
   791→		}
   792→
   793→		metric := query.Metric
   794→		qdata := res[0]
   795→		if qdata.Error != nil {
   796→			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
   797→		}
   798→
   799→		sanitizeStats(&qdata)
   800→
   801→		hostdata, ok := data[query.Hostname]
   802→		if !ok {
   803→			hostdata = make(map[string][]*schema.JobMetric)
   804→			data[query.Hostname] = hostdata
   805→		}
   806→
   807→		mc := archive.GetMetricConfig(cluster, metric)
   808→		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
   809→			Unit:     mc.Unit,
   810→			Timestep: mc.Timestep,
   811→			Series: []schema.Series{
   812→				{
   813→					Hostname: query.Hostname,
   814→					Data:     qdata.Data,
   815→					Statistics: schema.MetricStatistics{
   816→						Avg: float64(qdata.Avg),
   817→						Min: float64(qdata.Min),
   818→						Max: float64(qdata.Max),
   819→					},
   820→				},
   821→			},
   822→		})
   823→	}
   824→
   825→	if len(errors) != 0 {
   826→		/* Returns list of "partial errors" */
   827→		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   828→	}
   829→
   830→	return data, nil
   831→}
   832→
   833→// LoadNodeListData loads metric data for a list of nodes with full scope transformation support.
   834→//
   835→// This is the most flexible node data loading function, supporting arbitrary scopes and
   836→// resolution. Uses buildNodeQueries for proper scope transformation based on topology.
   837→//
   838→// Parameters:
   839→//   - cluster: Cluster name
   840→//   - subCluster: SubCluster name (empty string to infer from node names)
   841→//   - nodes: List of node hostnames
   842→//   - metrics: List of metric names
   843→//   - scopes: Requested metric scopes
   844→//   - resolution: Data resolution in seconds
   845→//   - from, to: Time range
   846→//   - ctx: Context (currently unused)
   847→//
   848→// Returns:
   849→//   - Map of hostname → JobData (metric → scope → JobMetric)
   850→//   - Error or partial error listing failed queries
   851→func (ccms *InternalMetricStore) LoadNodeListData(
   852→	cluster, subCluster string,
   853→	nodes []string,
   854→	metrics []string,
   855→	scopes []schema.MetricScope,
   856→	resolution int,
   857→	from, to time.Time,
   858→	ctx context.Context,
   859→) (map[string]schema.JobData, error) {
   860→	// Note: Order of node data is not guaranteed after this point
   861→	queries, assignedScope, err := buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution))
   862→	if err != nil {
   863→		cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
   864→		return nil, err
   865→	}
   866→
   867→	// Verify assignment is correct - log any inconsistencies for debugging
   868→	if len(queries) != len(assignedScope) {
   869→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
   870→			len(queries), len(assignedScope))
   871→	}
   872→
   873→	req := APIQueryRequest{
   874→		Cluster:   cluster,
   875→		Queries:   queries,
   876→		From:      from.Unix(),
   877→		To:        to.Unix(),
   878→		WithStats: true,
   879→		WithData:  true,
   880→	}
   881→
   882→	resBody, err := FetchData(req)
   883→	if err != nil {
   884→		cclog.Errorf("Error while fetching data : %s", err.Error())
   885→		return nil, err
   886→	}
   887→
   888→	var errors []string
   889→	data := make(map[string]schema.JobData)
   890→
   891→	// Add safety check for index out of range issues
   892→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   893→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   894→			len(req.Queries), len(resBody.Results), len(assignedScope))
   895→		if len(resBody.Results) > len(req.Queries) {
   896→			resBody.Results = resBody.Results[:len(req.Queries)]
   897→		}
   898→		if len(assignedScope) > len(req.Queries) {
   899→			assignedScope = assignedScope[:len(req.Queries)]
   900→		}
   901→	}
   902→
   903→	for i, row := range resBody.Results {
   904→		// Safety check to prevent index out of range errors
   905→		if i >= len(req.Queries) || i >= len(assignedScope) {
   906→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   907→				i, len(req.Queries), len(assignedScope))
   908→			continue
   909→		}
   910→
   911→		var query APIQuery
   912→		if resBody.Queries != nil {
   913→			if i < len(resBody.Queries) {
   914→				query = resBody.Queries[i]
   915→			} else {
   916→				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
   917→					i, len(resBody.Queries))
   918→				continue
   919→			}
   920→		} else {
   921→			query = req.Queries[i]
   922→		}
   923→
   924→		metric := query.Metric
   925→		scope := assignedScope[i]
   926→		mc := archive.GetMetricConfig(cluster, metric)
   927→		if mc == nil {
   928→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   929→			continue
   930→		}
   931→
   932→		res := mc.Timestep
   933→		if len(row) > 0 {
   934→			res = int(row[0].Resolution)
   935→		}
   936→
   937→		// Init Nested Map Data Structures If Not Found
   938→		hostData, ok := data[query.Hostname]
   939→		if !ok {
   940→			hostData = make(schema.JobData)
   941→			data[query.Hostname] = hostData
   942→		}
   943→
   944→		metricData, ok := hostData[metric]
   945→		if !ok {
   946→			metricData = make(map[schema.MetricScope]*schema.JobMetric)
   947→			data[query.Hostname][metric] = metricData
   948→		}
   949→
   950→		scopeData, ok := metricData[scope]
   951→		if !ok {
   952→			scopeData = &schema.JobMetric{
   953→				Unit:     mc.Unit,
   954→				Timestep: res,
   955→				Series:   make([]schema.Series, 0),
   956→			}
   957→			data[query.Hostname][metric][scope] = scopeData
   958→		}
   959→
   960→		for ndx, res := range row {
   961→			if res.Error != nil {
   962→				/* Build list for "partial errors", if any */
   963→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   964→				continue
   965→			}
   966→
   967→			id := (*string)(nil)
   968→			if query.Type != nil {
   969→				// Check if ndx is within the bounds of TypeIds slice
   970→				if ndx < len(query.TypeIds) {
   971→					id = new(string)
   972→					*id = query.TypeIds[ndx]
   973→				} else {
   974→					// Log the error but continue processing
   975→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   976→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   977→				}
   978→			}
   979→
   980→			sanitizeStats(&res)
   981→
   982→			scopeData.Series = append(scopeData.Series, schema.Series{
   983→				Hostname: query.Hostname,
   984→				ID:       id,
   985→				Statistics: schema.MetricStatistics{
   986→					Avg: float64(res.Avg),
   987→					Min: float64(res.Min),
   988→					Max: float64(res.Max),
   989→				},
   990→				Data: res.Data,
   991→			})
   992→		}
   993→	}
   994→
   995→	if len(errors) != 0 {
   996→		/* Returns list of "partial errors" */
   997→		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   998→	}
   999→
  1000→	return data, nil
  1001→}
  1002→
  1003→// buildNodeQueries constructs APIQuery structures for node-based queries with scope transformation.
  1004→//
  1005→// Similar to buildQueries but operates on node lists rather than job resources.
  1006→// Supports dynamic subcluster lookup when subCluster parameter is empty.
  1007→//
  1008→// Parameters:
  1009→//   - cluster: Cluster name
  1010→//   - subCluster: SubCluster name (empty = infer from node hostnames)
  1011→//   - nodes: List of node hostnames
  1012→//   - metrics: List of metric names
  1013→//   - scopes: Requested metric scopes
  1014→//   - resolution: Data resolution in seconds
  1015→//
  1016→// Returns:
  1017→//   - []APIQuery: List of queries to execute
  1018→//   - []schema.MetricScope: Assigned scope for each query
  1019→//   - error: Returns error if topology lookup fails or unhandled scope combination
  1020→func buildNodeQueries(
  1021→	cluster string,
  1022→	subCluster string,
  1023→	nodes []string,
  1024→	metrics []string,
  1025→	scopes []schema.MetricScope,
  1026→	resolution int64,
  1027→) ([]APIQuery, []schema.MetricScope, error) {
  1028→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
  1029→	assignedScope := []schema.MetricScope{}
  1030→
  1031→	// Get Topol before loop if subCluster given
  1032→	var subClusterTopol *schema.SubCluster
  1033→	var scterr error
  1034→	if subCluster != "" {
  1035→		subClusterTopol, scterr = archive.GetSubCluster(cluster, subCluster)
  1036→		if scterr != nil {
  1037→			cclog.Errorf("could not load cluster %s subCluster %s topology: %s", cluster, subCluster, scterr.Error())
  1038→			return nil, nil, scterr
  1039→		}
  1040→	}
  1041→
  1042→	for _, metric := range metrics {
  1043→		mc := archive.GetMetricConfig(cluster, metric)
  1044→		if mc == nil {
  1045→			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
  1046→			continue
  1047→		}
  1048→
  1049→		// Skip if metric is removed for subcluster
  1050→		if mc.SubClusters != nil {
  1051→			isRemoved := false
  1052→			for _, scConfig := range mc.SubClusters {
  1053→				if scConfig.Name == subCluster && scConfig.Remove {
  1054→					isRemoved = true
  1055→					break
  1056→				}
  1057→			}
  1058→			if isRemoved {
  1059→				continue
  1060→			}
  1061→		}
  1062→
  1063→		// Avoid duplicates using map for O(1) lookup
  1064→		handledScopes := make(map[schema.MetricScope]bool, 3)
  1065→
  1066→		for _, requestedScope := range scopes {
  1067→			nativeScope := mc.Scope
  1068→
  1069→			scope := nativeScope.Max(requestedScope)
  1070→			if handledScopes[scope] {
  1071→				continue
  1072→			}
  1073→			handledScopes[scope] = true
  1074→
  1075→			for _, hostname := range nodes {
  1076→
  1077→				// If no subCluster given, get it by node
  1078→				if subCluster == "" {
  1079→					subClusterName, scnerr := archive.GetSubClusterByNode(cluster, hostname)
  1080→					if scnerr != nil {
  1081→						return nil, nil, scnerr
  1082→					}
  1083→					subClusterTopol, scterr = archive.GetSubCluster(cluster, subClusterName)
  1084→					if scterr != nil {
  1085→						return nil, nil, scterr
  1086→					}
  1087→				}
  1088→
  1089→				// Always full node hwthread id list, no partial queries expected -> Use "topology.Node" directly where applicable
  1090→				// Always full accelerator id list, no partial queries expected -> Use "acceleratorIds" directly where applicable
  1091→				topology := subClusterTopol.Topology
  1092→				acceleratorIds := topology.GetAcceleratorIDs()
  1093→
  1094→				// Moved check here if metric matches hardware specs
  1095→				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
  1096→					continue
  1097→				}
  1098→
  1099→				// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
  1100→				if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
  1101→					if scope != schema.MetricScopeAccelerator {
  1102→						// Skip all other catched cases
  1103→						continue
  1104→					}
  1105→
  1106→					queries = append(queries, APIQuery{
  1107→						Metric:     metric,
  1108→						Hostname:   hostname,
  1109→						Aggregate:  false,
  1110→						Type:       &acceleratorString,
  1111→						TypeIds:    acceleratorIds,
  1112→						Resolution: resolution,
  1113→					})
  1114→					assignedScope = append(assignedScope, schema.MetricScopeAccelerator)
  1115→					continue
  1116→				}
  1117→
  1118→				// Accelerator -> Node
  1119→				if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
  1120→					if len(acceleratorIds) == 0 {
  1121→						continue
  1122→					}
  1123→
  1124→					queries = append(queries, APIQuery{
  1125→						Metric:     metric,
  1126→						Hostname:   hostname,
  1127→						Aggregate:  true,
  1128→						Type:       &acceleratorString,
  1129→						TypeIds:    acceleratorIds,
  1130→						Resolution: resolution,
  1131→					})
  1132→					assignedScope = append(assignedScope, scope)
  1133→					continue
  1134→				}
  1135→
  1136→				// HWThread -> HWThread
  1137→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
  1138→					queries = append(queries, APIQuery{
  1139→						Metric:     metric,
  1140→						Hostname:   hostname,
  1141→						Aggregate:  false,
  1142→						Type:       &hwthreadString,
  1143→						TypeIds:    intToStringSlice(topology.Node),
  1144→						Resolution: resolution,
  1145→					})
  1146→					assignedScope = append(assignedScope, scope)
  1147→					continue
  1148→				}
  1149→
  1150→				// HWThread -> Core
  1151→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
  1152→					cores, _ := topology.GetCoresFromHWThreads(topology.Node)
  1153→					for _, core := range cores {
  1154→						queries = append(queries, APIQuery{
  1155→							Metric:     metric,
  1156→							Hostname:   hostname,
  1157→							Aggregate:  true,
  1158→							Type:       &hwthreadString,
  1159→							TypeIds:    intToStringSlice(topology.Core[core]),
  1160→							Resolution: resolution,
  1161→						})
  1162→						assignedScope = append(assignedScope, scope)
  1163→					}
  1164→					continue
  1165→				}
  1166→
  1167→				// HWThread -> Socket
  1168→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
  1169→					sockets, _ := topology.GetSocketsFromHWThreads(topology.Node)
  1170→					for _, socket := range sockets {
  1171→						queries = append(queries, APIQuery{
  1172→							Metric:     metric,
  1173→							Hostname:   hostname,
  1174→							Aggregate:  true,
  1175→							Type:       &hwthreadString,
  1176→							TypeIds:    intToStringSlice(topology.Socket[socket]),
  1177→							Resolution: resolution,
  1178→						})
  1179→						assignedScope = append(assignedScope, scope)
  1180→					}
  1181→					continue
  1182→				}
  1183→
  1184→				// HWThread -> Node
  1185→				if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
  1186→					queries = append(queries, APIQuery{
  1187→						Metric:     metric,
  1188→						Hostname:   hostname,
  1189→						Aggregate:  true,
  1190→						Type:       &hwthreadString,
  1191→						TypeIds:    intToStringSlice(topology.Node),
  1192→						Resolution: resolution,
  1193→					})
  1194→					assignedScope = append(assignedScope, scope)
  1195→					continue
  1196→				}
  1197→
  1198→				// Core -> Core
  1199→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
  1200→					cores, _ := topology.GetCoresFromHWThreads(topology.Node)
  1201→					queries = append(queries, APIQuery{
  1202→						Metric:     metric,
  1203→						Hostname:   hostname,
  1204→						Aggregate:  false,
  1205→						Type:       &coreString,
  1206→						TypeIds:    intToStringSlice(cores),
  1207→						Resolution: resolution,
  1208→					})
  1209→					assignedScope = append(assignedScope, scope)
  1210→					continue
  1211→				}
  1212→
  1213→				// Core -> Socket
  1214→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
  1215→					sockets, _ := topology.GetSocketsFromCores(topology.Node)
  1216→					for _, socket := range sockets {
  1217→						queries = append(queries, APIQuery{
  1218→							Metric:     metric,
  1219→							Hostname:   hostname,
  1220→							Aggregate:  true,
  1221→							Type:       &coreString,
  1222→							TypeIds:    intToStringSlice(topology.Socket[socket]),
  1223→							Resolution: resolution,
  1224→						})
  1225→						assignedScope = append(assignedScope, scope)
  1226→					}
  1227→					continue
  1228→				}
  1229→
  1230→				// Core -> Node
  1231→				if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
  1232→					cores, _ := topology.GetCoresFromHWThreads(topology.Node)
  1233→					queries = append(queries, APIQuery{
  1234→						Metric:     metric,
  1235→						Hostname:   hostname,
  1236→						Aggregate:  true,
  1237→						Type:       &coreString,
  1238→						TypeIds:    intToStringSlice(cores),
  1239→						Resolution: resolution,
  1240→					})
  1241→					assignedScope = append(assignedScope, scope)
  1242→					continue
  1243→				}
  1244→
  1245→				// MemoryDomain -> MemoryDomain
  1246→				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
  1247→					sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node)
  1248→					queries = append(queries, APIQuery{
  1249→						Metric:     metric,
  1250→						Hostname:   hostname,
  1251→						Aggregate:  false,
  1252→						Type:       &memoryDomainString,
  1253→						TypeIds:    intToStringSlice(sockets),
  1254→						Resolution: resolution,
  1255→					})
  1256→					assignedScope = append(assignedScope, scope)
  1257→					continue
  1258→				}
  1259→
  1260→				// MemoryDomain -> Node
  1261→				if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
  1262→					sockets, _ := topology.GetMemoryDomainsFromHWThreads(topology.Node)
  1263→					queries = append(queries, APIQuery{
  1264→						Metric:     metric,
  1265→						Hostname:   hostname,
  1266→						Aggregate:  true,
  1267→						Type:       &memoryDomainString,
  1268→						TypeIds:    intToStringSlice(sockets),
  1269→						Resolution: resolution,
  1270→					})
  1271→					assignedScope = append(assignedScope, scope)
  1272→					continue
  1273→				}
  1274→
  1275→				// Socket -> Socket
  1276→				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
  1277→					sockets, _ := topology.GetSocketsFromHWThreads(topology.Node)
  1278→					queries = append(queries, APIQuery{
  1279→						Metric:     metric,
  1280→						Hostname:   hostname,
  1281→						Aggregate:  false,
  1282→						Type:       &socketString,
  1283→						TypeIds:    intToStringSlice(sockets),
  1284→						Resolution: resolution,
  1285→					})
  1286→					assignedScope = append(assignedScope, scope)
  1287→					continue
  1288→				}
  1289→
  1290→				// Socket -> Node
  1291→				if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
  1292→					sockets, _ := topology.GetSocketsFromHWThreads(topology.Node)
  1293→					queries = append(queries, APIQuery{
  1294→						Metric:     metric,
  1295→						Hostname:   hostname,
  1296→						Aggregate:  true,
  1297→						Type:       &socketString,
  1298→						TypeIds:    intToStringSlice(sockets),
  1299→						Resolution: resolution,
  1300→					})
  1301→					assignedScope = append(assignedScope, scope)
  1302→					continue
  1303→				}
  1304→
  1305→				// Node -> Node
  1306→				if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
  1307→					queries = append(queries, APIQuery{
  1308→						Metric:     metric,
  1309→						Hostname:   hostname,
  1310→						Resolution: resolution,
  1311→					})
  1312→					assignedScope = append(assignedScope, scope)
  1313→					continue
  1314→				}
  1315→
  1316→				return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
  1317→			}
  1318→		}
  1319→	}
  1320→
  1321→	return queries, assignedScope, nil
  1322→}
  1323→
  1324→// sanitizeStats converts NaN statistics to zero for JSON compatibility.
  1325→//
  1326→// schema.Float with NaN values cannot be properly JSON-encoded, so we convert
  1327→// NaN to 0. This loses the distinction between "no data" and "zero value",
  1328→// but maintains API compatibility.
  1329→func sanitizeStats(data *APIMetricData) {
  1330→	if data.Avg.IsNaN() {
  1331→		data.Avg = schema.Float(0)
  1332→	}
  1333→	if data.Min.IsNaN() {
  1334→		data.Min = schema.Float(0)
  1335→	}
  1336→	if data.Max.IsNaN() {
  1337→		data.Max = schema.Float(0)
  1338→	}
  1339→}
  1340→
  1341→// intToStringSlice converts a slice of integers to a slice of strings.
  1342→// Used to convert hardware thread/core/socket IDs from topology (int) to APIQuery TypeIds (string).
  1343→//
  1344→// Optimized to reuse a byte buffer for string conversion, reducing allocations.
  1345→func intToStringSlice(is []int) []string {
  1346→	if len(is) == 0 {
  1347→		return nil
  1348→	}
  1349→
  1350→	ss := make([]string, len(is))
  1351→	buf := make([]byte, 0, 16) // Reusable buffer for integer conversion
  1352→	for i, x := range is {
  1353→		buf = strconv.AppendInt(buf[:0], int64(x), 10)
  1354→		ss[i] = string(buf)
  1355→	}
  1356→	return ss
  1357→}
  1358→


/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store-queries.go
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstoreclient provides a client for querying the cc-metric-store time series database.
     7→//
     8→// The cc-metric-store is a high-performance time series database optimized for HPC metric data.
     9→// This client handles HTTP communication, query construction, scope transformations, and data retrieval
    10→// for job and node metrics across different metric scopes (node, socket, core, hwthread, accelerator).
    11→//
    12→// # Architecture
    13→//
    14→// The package is split into two main components:
    15→//   - Client Operations (cc-metric-store.go): HTTP client, request handling, data loading methods
    16→//   - Query Building (cc-metric-store-queries.go): Query construction and scope transformation logic
    17→//
    18→// # Basic Usage
    19→//
    20→//	store := NewCCMetricStore("http://localhost:8080", "jwt-token")
    21→//
    22→//	// Load job data
    23→//	jobData, err := store.LoadData(job, metrics, scopes, ctx, resolution)
    24→//	if err != nil {
    25→//	    log.Fatal(err)
    26→//	}
    27→//
    28→// # Metric Scopes
    29→//
    30→// The client supports hierarchical metric scopes that map to HPC hardware topology:
    31→//   - MetricScopeAccelerator: GPU/accelerator level metrics
    32→//   - MetricScopeHWThread: Hardware thread (SMT) level metrics
    33→//   - MetricScopeCore: CPU core level metrics
    34→//   - MetricScopeSocket: CPU socket level metrics
    35→//   - MetricScopeMemoryDomain: NUMA domain level metrics
    36→//   - MetricScopeNode: Full node level metrics
    37→//
    38→// The client automatically handles scope transformations, aggregating finer-grained metrics
    39→// to coarser scopes when needed (e.g., aggregating core metrics to socket level).
    40→//
    41→// # Error Handling
    42→//
    43→// The client supports partial errors - if some queries fail, it returns both the successful
    44→// data and an error listing the failed queries. This allows processing partial results
    45→// when some nodes or metrics are temporarily unavailable.
    46→//
    47→// # API Versioning
    48→//
    49→// The client uses cc-metric-store API v2, which includes support for:
    50→//   - Data resampling for bandwidth optimization
    51→//   - Multi-scope queries in a single request
    52→//   - Aggregation across hardware topology levels
    53→package metricstoreclient
    54→
    55→import (
    56→	"bufio"
    57→	"bytes"
    58→	"context"
    59→	"encoding/json"
    60→	"fmt"
    61→	"net/http"
    62→	"strings"
    63→	"time"
    64→
    65→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    66→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    67→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    68→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    69→)
    70→
    71→// CCMetricStore is the HTTP client for communicating with cc-metric-store.
    72→// It manages connection details, authentication, and provides methods for querying metrics.
    73→type CCMetricStore struct {
    74→	client        http.Client                 // HTTP client with 10-second timeout
    75→	jwt           string                      // JWT Bearer token for authentication
    76→	url           string                      // Base URL of cc-metric-store instance
    77→	queryEndpoint string                      // Full URL to query API endpoint
    78→	topologyCache map[string]*schema.Topology // cluster -> topology cache
    79→}
    80→
    81→// APIQueryRequest represents a request to the cc-metric-store query API.
    82→// It supports both explicit queries and "for-all-nodes" bulk queries.
    83→type APIQueryRequest struct {
    84→	Cluster     string     `json:"cluster"`       // Target cluster name
    85→	Queries     []APIQuery `json:"queries"`       // Explicit list of metric queries
    86→	ForAllNodes []string   `json:"for-all-nodes"` // Metrics to query for all nodes
    87→	From        int64      `json:"from"`          // Start time (Unix timestamp)
    88→	To          int64      `json:"to"`            // End time (Unix timestamp)
    89→	WithStats   bool       `json:"with-stats"`    // Include min/avg/max statistics
    90→	WithData    bool       `json:"with-data"`     // Include time series data points
    91→}
    92→
    93→// APIQuery specifies a single metric query with optional scope filtering.
    94→// Type and TypeIds define the hardware scope (core, socket, accelerator, etc.).
    95→type APIQuery struct {
    96→	Type       *string  `json:"type,omitempty"`        // Scope type (e.g., "core", "socket")
    97→	SubType    *string  `json:"subtype,omitempty"`     // Sub-scope type (reserved for future use)
    98→	Metric     string   `json:"metric"`                // Metric name
    99→	Hostname   string   `json:"host"`                  // Target hostname
   100→	Resolution int      `json:"resolution"`            // Data resolution in seconds (0 = native)
   101→	TypeIds    []string `json:"type-ids,omitempty"`    // IDs for the scope type (e.g., core IDs)
   102→	SubTypeIds []string `json:"subtype-ids,omitempty"` // IDs for sub-scope (reserved)
   103→	Aggregate  bool     `json:"aggreg"`                // Aggregate across TypeIds
   104→}
   105→
   106→// APIQueryResponse contains the results from a cc-metric-store query.
   107→// Results align with the Queries slice by index.
   108→type APIQueryResponse struct {
   109→	Queries []APIQuery        `json:"queries,omitempty"` // Echoed queries (for bulk requests)
   110→	Results [][]APIMetricData `json:"results"`           // Result data, indexed by query
   111→}
   112→
   113→// APIMetricData represents time series data and statistics for a single metric series.
   114→// Error is set if this particular series failed to load.
   115→type APIMetricData struct {
   116→	Error      *string        `json:"error"`      // Error message if query failed
   117→	Data       []schema.Float `json:"data"`       // Time series data points
   118→	From       int64          `json:"from"`       // Actual start time of data
   119→	To         int64          `json:"to"`         // Actual end time of data
   120→	Resolution int            `json:"resolution"` // Actual resolution of data in seconds
   121→	Avg        schema.Float   `json:"avg"`        // Average value across time range
   122→	Min        schema.Float   `json:"min"`        // Minimum value in time range
   123→	Max        schema.Float   `json:"max"`        // Maximum value in time range
   124→}
   125→
   126→// NewCCMetricStore creates and initializes a new (external) CCMetricStore client.
   127→// The url parameter should include the protocol and port (e.g., "http://localhost:8080").
   128→// The token parameter is a JWT used for Bearer authentication; pass empty string if auth is disabled.
   129→func NewCCMetricStore(url string, token string) *CCMetricStore {
   130→	return &CCMetricStore{
   131→		url:           url,
   132→		queryEndpoint: fmt.Sprintf("%s/api/query", url),
   133→		jwt:           token,
   134→		client: http.Client{
   135→			Timeout: 10 * time.Second,
   136→		},
   137→		topologyCache: make(map[string]*schema.Topology),
   138→	}
   139→}
   140→
   141→// doRequest executes an HTTP POST request to the cc-metric-store query API.
   142→// It handles JSON encoding/decoding, authentication, and API versioning.
   143→// The request body is automatically closed to prevent resource leaks.
   144→func (ccms *CCMetricStore) doRequest(
   145→	ctx context.Context,
   146→	body *APIQueryRequest,
   147→) (*APIQueryResponse, error) {
   148→	buf := &bytes.Buffer{}
   149→	if err := json.NewEncoder(buf).Encode(body); err != nil {
   150→		cclog.Errorf("Error while encoding request body: %s", err.Error())
   151→		return nil, err
   152→	}
   153→
   154→	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf)
   155→	if err != nil {
   156→		cclog.Errorf("Error while building request body: %s", err.Error())
   157→		return nil, err
   158→	}
   159→	if ccms.jwt != "" {
   160→		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   161→	}
   162→
   163→	// versioning the cc-metric-store query API.
   164→	// v2 = data with resampling
   165→	// v1 = data without resampling
   166→	q := req.URL.Query()
   167→	q.Add("version", "v2")
   168→	req.URL.RawQuery = q.Encode()
   169→
   170→	res, err := ccms.client.Do(req)
   171→	if err != nil {
   172→		cclog.Errorf("Error while performing request: %s", err.Error())
   173→		return nil, err
   174→	}
   175→	defer res.Body.Close()
   176→
   177→	if res.StatusCode != http.StatusOK {
   178→		return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status)
   179→	}
   180→
   181→	var resBody APIQueryResponse
   182→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
   183→		cclog.Errorf("Error while decoding result body: %s", err.Error())
   184→		return nil, err
   185→	}
   186→
   187→	return &resBody, nil
   188→}
   189→
   190→// getTopology returns the topology for a given cluster and subcluster, caching it if not already present
   191→func (ccms *CCMetricStore) getTopology(cluster, subCluster string) (*schema.Topology, error) {
   192→	cacheKey := fmt.Sprintf("%s:%s", cluster, subCluster)
   193→	if topology, ok := ccms.topologyCache[cacheKey]; ok {
   194→		return topology, nil
   195→	}
   196→
   197→	subcluster, err := archive.GetSubCluster(cluster, subCluster)
   198→	if err != nil {
   199→		return nil, err
   200→	}
   201→
   202→	ccms.topologyCache[cacheKey] = &subcluster.Topology
   203→	return &subcluster.Topology, nil
   204→}
   205→
   206→// getTopologyByNode returns the topology for a given cluster and node, caching it if not already present
   207→func (ccms *CCMetricStore) getTopologyByNode(cluster, node string) (*schema.Topology, error) {
   208→	subCluster, err := archive.GetSubClusterByNode(cluster, node)
   209→	if err != nil {
   210→		return nil, err
   211→	}
   212→
   213→	return ccms.getTopology(cluster, subCluster)
   214→}
   215→
   216→// LoadData retrieves time series data and statistics for the specified job and metrics.
   217→// It queries data for the job's time range and resources, handling scope transformations automatically.
   218→//
   219→// Parameters:
   220→//   - job: Job metadata including cluster, time range, and allocated resources
   221→//   - metrics: List of metric names to retrieve
   222→//   - scopes: Requested metric scopes (node, socket, core, etc.)
   223→//   - ctx: Context for cancellation and timeouts
   224→//   - resolution: Data resolution in seconds (0 for native resolution)
   225→//
   226→// Returns JobData organized as: metric -> scope -> series list.
   227→// Supports partial errors: returns available data even if some queries fail.
   228→func (ccms *CCMetricStore) LoadData(
   229→	job *schema.Job,
   230→	metrics []string,
   231→	scopes []schema.MetricScope,
   232→	ctx context.Context,
   233→	resolution int,
   234→) (schema.JobData, error) {
   235→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
   236→	if err != nil {
   237→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   238→		return nil, err
   239→	}
   240→
   241→	// Verify assignment is correct - log any inconsistencies for debugging
   242→	if len(queries) != len(assignedScope) {
   243→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
   244→			len(queries), len(assignedScope))
   245→	}
   246→
   247→	req := APIQueryRequest{
   248→		Cluster:   job.Cluster,
   249→		From:      job.StartTime,
   250→		To:        job.StartTime + int64(job.Duration),
   251→		Queries:   queries,
   252→		WithStats: true,
   253→		WithData:  true,
   254→	}
   255→
   256→	resBody, err := ccms.doRequest(ctx, &req)
   257→	if err != nil {
   258→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   259→		return nil, err
   260→	}
   261→
   262→	var errors []string
   263→	jobData := make(schema.JobData)
   264→
   265→	// Add safety check for potential index out of range errors
   266→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   267→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   268→			len(req.Queries), len(resBody.Results), len(assignedScope))
   269→		if len(resBody.Results) > len(req.Queries) {
   270→			resBody.Results = resBody.Results[:len(req.Queries)]
   271→		}
   272→		if len(assignedScope) > len(req.Queries) {
   273→			assignedScope = assignedScope[:len(req.Queries)]
   274→		}
   275→	}
   276→
   277→	for i, row := range resBody.Results {
   278→		// Safety check to prevent index out of range errors
   279→		if i >= len(req.Queries) || i >= len(assignedScope) {
   280→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   281→				i, len(req.Queries), len(assignedScope))
   282→			continue
   283→		}
   284→
   285→		query := req.Queries[i]
   286→		metric := query.Metric
   287→		scope := assignedScope[i]
   288→		mc := archive.GetMetricConfig(job.Cluster, metric)
   289→
   290→		if mc == nil {
   291→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   292→			continue
   293→		}
   294→
   295→		if _, ok := jobData[metric]; !ok {
   296→			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
   297→		}
   298→
   299→		res := mc.Timestep
   300→		if len(row) > 0 {
   301→			res = row[0].Resolution
   302→		}
   303→
   304→		jobMetric, ok := jobData[metric][scope]
   305→		if !ok {
   306→			jobMetric = &schema.JobMetric{
   307→				Unit:     mc.Unit,
   308→				Timestep: res,
   309→				Series:   make([]schema.Series, 0),
   310→			}
   311→			jobData[metric][scope] = jobMetric
   312→		}
   313→
   314→		for ndx, res := range row {
   315→			if res.Error != nil {
   316→				/* Build list for "partial errors", if any */
   317→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   318→				continue
   319→			}
   320→
   321→			id := (*string)(nil)
   322→			if query.Type != nil {
   323→				// Check if ndx is within the bounds of TypeIds slice
   324→				if ndx < len(query.TypeIds) {
   325→					id = new(string)
   326→					*id = query.TypeIds[ndx]
   327→				} else {
   328→					// Log the error but continue processing
   329→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   330→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   331→				}
   332→			}
   333→
   334→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   335→
   336→			jobMetric.Series = append(jobMetric.Series, schema.Series{
   337→				Hostname: query.Hostname,
   338→				ID:       id,
   339→				Statistics: schema.MetricStatistics{
   340→					Avg: float64(res.Avg),
   341→					Min: float64(res.Min),
   342→					Max: float64(res.Max),
   343→				},
   344→				Data: res.Data,
   345→			})
   346→		}
   347→
   348→		// So that one can later check len(jobData):
   349→		if len(jobMetric.Series) == 0 {
   350→			delete(jobData[metric], scope)
   351→			if len(jobData[metric]) == 0 {
   352→				delete(jobData, metric)
   353→			}
   354→		}
   355→	}
   356→
   357→	if len(errors) != 0 {
   358→		/* Returns list for "partial errors" */
   359→		return jobData, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   360→	}
   361→	return jobData, nil
   362→}
   363→
   364→// LoadStats retrieves min/avg/max statistics for job metrics at node scope.
   365→// This is faster than LoadData when only statistical summaries are needed (no time series data).
   366→//
   367→// Returns statistics organized as: metric -> hostname -> statistics.
   368→func (ccms *CCMetricStore) LoadStats(
   369→	job *schema.Job,
   370→	metrics []string,
   371→	ctx context.Context,
   372→) (map[string]map[string]schema.MetricStatistics, error) {
   373→	queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization?
   374→	if err != nil {
   375→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
   376→		return nil, err
   377→	}
   378→
   379→	req := APIQueryRequest{
   380→		Cluster:   job.Cluster,
   381→		From:      job.StartTime,
   382→		To:        job.StartTime + int64(job.Duration),
   383→		Queries:   queries,
   384→		WithStats: true,
   385→		WithData:  false,
   386→	}
   387→
   388→	resBody, err := ccms.doRequest(ctx, &req)
   389→	if err != nil {
   390→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   391→		return nil, err
   392→	}
   393→
   394→	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
   395→	for i, res := range resBody.Results {
   396→		if len(res) == 0 {
   397→			// No Data Found For Metric, Logged in FetchData to Warn
   398→			continue
   399→		}
   400→		query := req.Queries[i]
   401→		metric := query.Metric
   402→		data := res[0]
   403→		if data.Error != nil {
   404→			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
   405→			continue
   406→		}
   407→
   408→		metricdata, ok := stats[metric]
   409→		if !ok {
   410→			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
   411→			stats[metric] = metricdata
   412→		}
   413→
   414→		if hasNaNStats(data.Avg, data.Min, data.Max) {
   415→			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
   416→			continue
   417→		}
   418→
   419→		metricdata[query.Hostname] = schema.MetricStatistics{
   420→			Avg: float64(data.Avg),
   421→			Min: float64(data.Min),
   422→			Max: float64(data.Max),
   423→		}
   424→	}
   425→
   426→	return stats, nil
   427→}
   428→
   429→// LoadScopedStats retrieves statistics for job metrics across multiple scopes.
   430→// Used for the Job-View Statistics Table to display per-scope breakdowns.
   431→//
   432→// Returns statistics organized as: metric -> scope -> list of scoped statistics.
   433→// Each scoped statistic includes hostname, hardware ID (if applicable), and min/avg/max values.
   434→func (ccms *CCMetricStore) LoadScopedStats(
   435→	job *schema.Job,
   436→	metrics []string,
   437→	scopes []schema.MetricScope,
   438→	ctx context.Context,
   439→) (schema.ScopedJobStats, error) {
   440→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
   441→	if err != nil {
   442→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   443→		return nil, err
   444→	}
   445→
   446→	req := APIQueryRequest{
   447→		Cluster:   job.Cluster,
   448→		From:      job.StartTime,
   449→		To:        job.StartTime + int64(job.Duration),
   450→		Queries:   queries,
   451→		WithStats: true,
   452→		WithData:  false,
   453→	}
   454→
   455→	resBody, err := ccms.doRequest(ctx, &req)
   456→	if err != nil {
   457→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   458→		return nil, err
   459→	}
   460→
   461→	var errors []string
   462→	scopedJobStats := make(schema.ScopedJobStats)
   463→
   464→	for i, row := range resBody.Results {
   465→		query := req.Queries[i]
   466→		metric := query.Metric
   467→		scope := assignedScope[i]
   468→
   469→		if _, ok := scopedJobStats[metric]; !ok {
   470→			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
   471→		}
   472→
   473→		if _, ok := scopedJobStats[metric][scope]; !ok {
   474→			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
   475→		}
   476→
   477→		for ndx, res := range row {
   478→			if res.Error != nil {
   479→				/* Build list for "partial errors", if any */
   480→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   481→				continue
   482→			}
   483→
   484→			id := (*string)(nil)
   485→			if query.Type != nil {
   486→				// Check if ndx is within the bounds of TypeIds slice
   487→				if ndx < len(query.TypeIds) {
   488→					id = new(string)
   489→					*id = query.TypeIds[ndx]
   490→				} else {
   491→					// Log the error but continue processing
   492→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   493→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   494→				}
   495→			}
   496→
   497→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   498→
   499→			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
   500→				Hostname: query.Hostname,
   501→				ID:       id,
   502→				Data: &schema.MetricStatistics{
   503→					Avg: float64(res.Avg),
   504→					Min: float64(res.Min),
   505→					Max: float64(res.Max),
   506→				},
   507→			})
   508→		}
   509→
   510→		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
   511→		if len(scopedJobStats[metric][scope]) == 0 {
   512→			delete(scopedJobStats[metric], scope)
   513→			if len(scopedJobStats[metric]) == 0 {
   514→				delete(scopedJobStats, metric)
   515→			}
   516→		}
   517→	}
   518→
   519→	if len(errors) != 0 {
   520→		/* Returns list for "partial errors" */
   521→		return scopedJobStats, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   522→	}
   523→	return scopedJobStats, nil
   524→}
   525→
   526→// LoadNodeData retrieves current metric data for specified nodes in a cluster.
   527→// Used for the Systems-View Node-Overview to display real-time node status.
   528→//
   529→// If nodes is nil, queries all metrics for all nodes in the cluster (bulk query).
   530→// Returns data organized as: hostname -> metric -> list of JobMetric (with time series and stats).
   531→func (ccms *CCMetricStore) LoadNodeData(
   532→	cluster string,
   533→	metrics, nodes []string,
   534→	scopes []schema.MetricScope,
   535→	from, to time.Time,
   536→	ctx context.Context,
   537→) (map[string]map[string][]*schema.JobMetric, error) {
   538→	req := APIQueryRequest{
   539→		Cluster:   cluster,
   540→		From:      from.Unix(),
   541→		To:        to.Unix(),
   542→		WithStats: true,
   543→		WithData:  true,
   544→	}
   545→
   546→	if nodes == nil {
   547→		req.ForAllNodes = append(req.ForAllNodes, metrics...)
   548→	} else {
   549→		for _, node := range nodes {
   550→			for _, metric := range metrics {
   551→				req.Queries = append(req.Queries, APIQuery{
   552→					Hostname:   node,
   553→					Metric:     metric,
   554→					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
   555→				})
   556→			}
   557→		}
   558→	}
   559→
   560→	resBody, err := ccms.doRequest(ctx, &req)
   561→	if err != nil {
   562→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   563→		return nil, err
   564→	}
   565→
   566→	var errors []string
   567→	data := make(map[string]map[string][]*schema.JobMetric)
   568→	for i, res := range resBody.Results {
   569→		if len(res) == 0 {
   570→			// No Data Found For Metric, Logged in FetchData to Warn
   571→			continue
   572→		}
   573→
   574→		var query APIQuery
   575→		if resBody.Queries != nil {
   576→			query = resBody.Queries[i]
   577→		} else {
   578→			query = req.Queries[i]
   579→		}
   580→
   581→		metric := query.Metric
   582→		qdata := res[0]
   583→		if qdata.Error != nil {
   584→			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
   585→		}
   586→
   587→		sanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)
   588→
   589→		hostdata, ok := data[query.Hostname]
   590→		if !ok {
   591→			hostdata = make(map[string][]*schema.JobMetric)
   592→			data[query.Hostname] = hostdata
   593→		}
   594→
   595→		mc := archive.GetMetricConfig(cluster, metric)
   596→		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
   597→			Unit:     mc.Unit,
   598→			Timestep: mc.Timestep,
   599→			Series: []schema.Series{
   600→				{
   601→					Hostname: query.Hostname,
   602→					Data:     qdata.Data,
   603→					Statistics: schema.MetricStatistics{
   604→						Avg: float64(qdata.Avg),
   605→						Min: float64(qdata.Min),
   606→						Max: float64(qdata.Max),
   607→					},
   608→				},
   609→			},
   610→		})
   611→	}
   612→
   613→	if len(errors) != 0 {
   614→		/* Returns list of "partial errors" */
   615→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   616→	}
   617→
   618→	return data, nil
   619→}
   620→
   621→// LoadNodeListData retrieves paginated node metrics for the Systems-View Node-List.
   622→//
   623→// Supports filtering by subcluster and node name pattern. The nodeFilter performs
   624→// substring matching on hostnames.
   625→//
   626→// Returns:
   627→//   - Node data organized as: hostname -> JobData (metric -> scope -> series)
   628→//   - Total node count (before pagination)
   629→//   - HasNextPage flag indicating if more pages are available
   630→//   - Error (may be partial error with some data returned)
   631→func (ccms *CCMetricStore) LoadNodeListData(
   632→	cluster, subCluster string,
   633→	nodes []string,
   634→	metrics []string,
   635→	scopes []schema.MetricScope,
   636→	resolution int,
   637→	from, to time.Time,
   638→	ctx context.Context,
   639→) (map[string]schema.JobData, error) {
   640→	queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
   641→	if err != nil {
   642→		cclog.Errorf("Error while building node queries for Cluster %s, SubCluster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
   643→		return nil, err
   644→	}
   645→
   646→	// Verify assignment is correct - log any inconsistencies for debugging
   647→	if len(queries) != len(assignedScope) {
   648→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
   649→			len(queries), len(assignedScope))
   650→	}
   651→
   652→	req := APIQueryRequest{
   653→		Cluster:   cluster,
   654→		Queries:   queries,
   655→		From:      from.Unix(),
   656→		To:        to.Unix(),
   657→		WithStats: true,
   658→		WithData:  true,
   659→	}
   660→
   661→	resBody, err := ccms.doRequest(ctx, &req)
   662→	if err != nil {
   663→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   664→		return nil, err
   665→	}
   666→
   667→	var errors []string
   668→	data := make(map[string]schema.JobData)
   669→
   670→	// Add safety check for index out of range issues
   671→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   672→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   673→			len(req.Queries), len(resBody.Results), len(assignedScope))
   674→		if len(resBody.Results) > len(req.Queries) {
   675→			resBody.Results = resBody.Results[:len(req.Queries)]
   676→		}
   677→		if len(assignedScope) > len(req.Queries) {
   678→			assignedScope = assignedScope[:len(req.Queries)]
   679→		}
   680→	}
   681→
   682→	for i, row := range resBody.Results {
   683→		// Safety check to prevent index out of range errors
   684→		if i >= len(req.Queries) || i >= len(assignedScope) {
   685→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   686→				i, len(req.Queries), len(assignedScope))
   687→			continue
   688→		}
   689→
   690→		var query APIQuery
   691→		if resBody.Queries != nil {
   692→			if i < len(resBody.Queries) {
   693→				query = resBody.Queries[i]
   694→			} else {
   695→				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
   696→					i, len(resBody.Queries))
   697→				continue
   698→			}
   699→		} else {
   700→			query = req.Queries[i]
   701→		}
   702→
   703→		metric := query.Metric
   704→		scope := assignedScope[i]
   705→		mc := archive.GetMetricConfig(cluster, metric)
   706→		if mc == nil {
   707→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   708→			continue
   709→		}
   710→
   711→		res := mc.Timestep
   712→		if len(row) > 0 {
   713→			res = row[0].Resolution
   714→		}
   715→
   716→		// Init Nested Map Data Structures If Not Found
   717→		hostData, ok := data[query.Hostname]
   718→		if !ok {
   719→			hostData = make(schema.JobData)
   720→			data[query.Hostname] = hostData
   721→		}
   722→
   723→		metricData, ok := hostData[metric]
   724→		if !ok {
   725→			metricData = make(map[schema.MetricScope]*schema.JobMetric)
   726→			data[query.Hostname][metric] = metricData
   727→		}
   728→
   729→		scopeData, ok := metricData[scope]
   730→		if !ok {
   731→			scopeData = &schema.JobMetric{
   732→				Unit:     mc.Unit,
   733→				Timestep: res,
   734→				Series:   make([]schema.Series, 0),
   735→			}
   736→			data[query.Hostname][metric][scope] = scopeData
   737→		}
   738→
   739→		for ndx, res := range row {
   740→			if res.Error != nil {
   741→				/* Build list for "partial errors", if any */
   742→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   743→				continue
   744→			}
   745→
   746→			id := (*string)(nil)
   747→			if query.Type != nil {
   748→				// Check if ndx is within the bounds of TypeIds slice
   749→				if ndx < len(query.TypeIds) {
   750→					id = new(string)
   751→					*id = query.TypeIds[ndx]
   752→				} else {
   753→					// Log the error but continue processing
   754→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   755→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   756→				}
   757→			}
   758→
   759→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   760→
   761→			scopeData.Series = append(scopeData.Series, schema.Series{
   762→				Hostname: query.Hostname,
   763→				ID:       id,
   764→				Statistics: schema.MetricStatistics{
   765→					Avg: float64(res.Avg),
   766→					Min: float64(res.Min),
   767→					Max: float64(res.Max),
   768→				},
   769→				Data: res.Data,
   770→			})
   771→		}
   772→	}
   773→
   774→	if len(errors) != 0 {
   775→		/* Returns list of "partial errors" */
   776→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   777→	}
   778→
   779→	return data, nil
   780→}
   781→
   782→// HealthCheck queries the external cc-metric-store's health check endpoint.
   783→// It sends a HealthCheckReq as the request body to /api/healthcheck and
   784→// returns the per-node health check results.
   785→func (ccms *CCMetricStore) HealthCheck(cluster string,
   786→	nodes []string, metrics []string,
   787→) (map[string]metricstore.HealthCheckResult, error) {
   788→	req := metricstore.HealthCheckReq{
   789→		Cluster:     cluster,
   790→		Nodes:       nodes,
   791→		MetricNames: metrics,
   792→	}
   793→
   794→	buf := &bytes.Buffer{}
   795→	if err := json.NewEncoder(buf).Encode(req); err != nil {
   796→		cclog.Errorf("Error while encoding health check request body: %s", err.Error())
   797→		return nil, err
   798→	}
   799→
   800→	endpoint := fmt.Sprintf("%s/api/healthcheck", ccms.url)
   801→	httpReq, err := http.NewRequest(http.MethodGet, endpoint, buf)
   802→	if err != nil {
   803→		cclog.Errorf("Error while building health check request: %s", err.Error())
   804→		return nil, err
   805→	}
   806→	if ccms.jwt != "" {
   807→		httpReq.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   808→	}
   809→
   810→	res, err := ccms.client.Do(httpReq)
   811→	if err != nil {
   812→		cclog.Errorf("Error while performing health check request: %s", err.Error())
   813→		return nil, err
   814→	}
   815→	defer res.Body.Close()
   816→
   817→	if res.StatusCode != http.StatusOK {
   818→		return nil, fmt.Errorf("'%s': HTTP Status: %s", endpoint, res.Status)
   819→	}
   820→
   821→	var results map[string]metricstore.HealthCheckResult
   822→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&results); err != nil {
   823→		cclog.Errorf("Error while decoding health check response: %s", err.Error())
   824→		return nil, err
   825→	}
   826→
   827→	return results, nil
   828→}
   829→
   830→// sanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
   831→// Regular float64 values cannot be JSONed when NaN.
   832→func sanitizeStats(avg, min, max *schema.Float) {
   833→	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
   834→		*avg = schema.Float(0)
   835→		*min = schema.Float(0)
   836→		*max = schema.Float(0)
   837→	}
   838→}
   839→
   840→// hasNaNStats returns true if any of the statistics contain NaN values.
   841→func hasNaNStats(avg, min, max schema.Float) bool {
   842→	return avg.IsNaN() || min.IsNaN() || max.IsNaN()
   843→}
   844→


pkg/metricstore/api.go:83:type APIQuery struct {
pkg/metricstore/api.go-84-	Type        *string      `json:"type,omitempty"`
pkg/metricstore/api.go-85-	SubType     *string      `json:"subtype,omitempty"`
pkg/metricstore/api.go-86-	Metric      string       `json:"metric"`
pkg/metricstore/api.go-87-	Hostname    string       `json:"host"`
pkg/metricstore/api.go-88-	Resolution  int64        `json:"resolution"`
pkg/metricstore/api.go-89-	TypeIds     []string     `json:"type-ids,omitempty"`
pkg/metricstore/api.go-90-	SubTypeIds  []string     `json:"subtype-ids,omitempty"`
pkg/metricstore/api.go-91-	ScaleFactor schema.Float `json:"scale-by,omitempty"`
pkg/metricstore/api.go-92-	Aggregate   bool         `json:"aggreg"`
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-93-}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-94-
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-95-// AddStats computes and populates the Avg, Min, and Max fields from the Data array.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-96-//
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-97-// NaN values in the data are ignored during computation. If all values are NaN,
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-98-// the statistics fields will be set to NaN.
--
internal/metricstoreclient/cc-metric-store.go:95:type APIQuery struct {
internal/metricstoreclient/cc-metric-store.go-96-	Type       *string  `json:"type,omitempty"`        // Scope type (e.g., "core", "socket")
internal/metricstoreclient/cc-metric-store.go-97-	SubType    *string  `json:"subtype,omitempty"`     // Sub-scope type (reserved for future use)
internal/metricstoreclient/cc-metric-store.go-98-	Metric     string   `json:"metric"`                // Metric name
internal/metricstoreclient/cc-metric-store.go-99-	Hostname   string   `json:"host"`                  // Target hostname
internal/metricstoreclient/cc-metric-store.go-100-	Resolution int      `json:"resolution"`            // Data resolution in seconds (0 = native)
internal/metricstoreclient/cc-metric-store.go-101-	TypeIds    []string `json:"type-ids,omitempty"`    // IDs for the scope type (e.g., core IDs)
internal/metricstoreclient/cc-metric-store.go-102-	SubTypeIds []string `json:"subtype-ids,omitempty"` // IDs for sub-scope (reserved)
internal/metricstoreclient/cc-metric-store.go-103-	Aggregate  bool     `json:"aggreg"`                // Aggregate across TypeIds
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-104-}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-105-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-106-// APIQueryResponse contains the results from a cc-metric-store query.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-107-// Results align with the Queries slice by index.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-108-type APIQueryResponse struct {
internal/metricstoreclient/cc-metric-store.go-109-	Queries []APIQuery        `json:"queries,omitempty"` // Echoed queries (for bulk requests)
internal/metricstoreclient/cc-metric-store.go-110-	Results [][]APIMetricData `json:"results"`           // Result data, indexed by query


    60→	WithData    bool       `json:"with-data"`
    61→	WithPadding bool       `json:"with-padding"`
    62→}
    63→
    64→// APIQueryResponse represents the response to an APIQueryRequest.
    65→//
    66→// Results is a 2D array where each outer element corresponds to a query,
    67→// and each inner element corresponds to a selector within that query
    68→// (e.g., multiple CPUs or cores).
    69→type APIQueryResponse struct {
    70→	Queries []APIQuery        `json:"queries,omitempty"`
    71→	Results [][]APIMetricData `json:"results"`
    72→}
    73→
    74→// APIQuery represents a single metric query with optional hierarchical selectors.
    75→//
    76→// The hierarchical selection works as follows:
    77→//   - Hostname: The node to query
    78→//   - Type + TypeIds: First level of hierarchy (e.g., "cpu" + ["0", "1", "2"])
    79→//   - SubType + SubTypeIds: Second level of hierarchy (e.g., "core" + ["0", "1"])
    80→//
    81→// If Aggregate is true, data from multiple type/subtype IDs will be aggregated according
    82→// to the metric's aggregation strategy. Otherwise, separate results are returned for each combination.
    83→type APIQuery struct {
    84→	Type        *string      `json:"type,omitempty"`
    85→	SubType     *string      `json:"subtype,omitempty"`
    86→	Metric      string       `json:"metric"`
    87→	Hostname    string       `json:"host"`
    88→	Resolution  int64        `json:"resolution"`
    89→	TypeIds     []string     `json:"type-ids,omitempty"`
    90→	SubTypeIds  []string     `json:"subtype-ids,omitempty"`
    91→	ScaleFactor schema.Float `json:"scale-by,omitempty"`
    92→	Aggregate   bool         `json:"aggreg"`
    93→}
    94→
    95→// AddStats computes and populates the Avg, Min, and Max fields from the Data array.
    96→//
    97→// NaN values in the data are ignored during computation. If all values are NaN,
    98→// the statistics fields will be set to NaN.
    99→//
   100→// TODO: Optimize this, just like the stats endpoint!
   101→func (data *APIMetricData) AddStats() {
   102→	n := 0
   103→	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
   104→	for _, x := range data.Data {
   105→		if x.IsNaN() {
   106→			continue
   107→		}
   108→
   109→		n += 1
   110→		sum += float64(x)
   111→		min = math.Min(min, float64(x))
   112→		max = math.Max(max, float64(x))
   113→	}
   114→
   115→	if n > 0 {
   116→		avg := sum / float64(n)
   117→		data.Avg = schema.Float(avg)
   118→		data.Min = schema.Float(min)
   119→		data.Max = schema.Float(max)
   120→	} else {
   121→		data.Avg, data.Min, data.Max = schema.NaN, schema.NaN, schema.NaN
   122→	}
   123→}
   124→
   125→// ScaleBy multiplies all data points and statistics by the given factor.
   126→//
   127→// This is commonly used for unit conversion (e.g., bytes to gigabytes).
   128→// Scaling by 0 or 1 is a no-op for performance reasons.
   129→func (data *APIMetricData) ScaleBy(f schema.Float) {
   130→	if f == 0 || f == 1 {
   131→		return
   132→	}
   133→
   134→	data.Avg *= f
   135→	data.Min *= f
   136→	data.Max *= f
   137→	for i := 0; i < len(data.Data); i++ {
   138→		data.Data[i] *= f
   139→	}


Found 1 file
internal/metricstoreclient/cc-metric-store-queries.go


/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store-queries.go-449-	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket {
internal/metricstoreclient/cc-metric-store-queries.go-450-		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
internal/metricstoreclient/cc-metric-store-queries.go:451:		socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store-queries.go-452-		if err != nil {
internal/metricstoreclient/cc-metric-store-queries.go-453-			cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store-queries.go-454-			// Rare Error Case -> Still Continue -> Return Empty Slices
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store-queries.go-455-			return queries, scopes
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store-queries.go-456-		}


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore_test.go
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/parquetArchive_test.go


No files found


pkg/metricstore/api.go:35:type APIMetricData struct {
pkg/metricstore/api.go-36-	Error      *string           `json:"error,omitempty"`
pkg/metricstore/api.go-37-	Data       schema.FloatArray `json:"data,omitempty"`
pkg/metricstore/api.go-38-	From       int64             `json:"from"`
pkg/metricstore/api.go-39-	To         int64             `json:"to"`
pkg/metricstore/api.go-40-	Resolution int64             `json:"resolution"`
pkg/metricstore/api.go-41-	Avg        schema.Float      `json:"avg"`
pkg/metricstore/api.go-42-	Min        schema.Float      `json:"min"`
pkg/metricstore/api.go-43-	Max        schema.Float      `json:"max"`
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-44-}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-45-
--
internal/metricstoreclient/cc-metric-store.go:115:type APIMetricData struct {
internal/metricstoreclient/cc-metric-store.go-116-	Error      *string        `json:"error"`      // Error message if query failed
internal/metricstoreclient/cc-metric-store.go-117-	Data       []schema.Float `json:"data"`       // Time series data points
internal/metricstoreclient/cc-metric-store.go-118-	From       int64          `json:"from"`       // Actual start time of data
internal/metricstoreclient/cc-metric-store.go-119-	To         int64          `json:"to"`         // Actual end time of data
internal/metricstoreclient/cc-metric-store.go-120-	Resolution int            `json:"resolution"` // Actual resolution of data in seconds
internal/metricstoreclient/cc-metric-store.go-121-	Avg        schema.Float   `json:"avg"`        // Average value across time range
internal/metricstoreclient/cc-metric-store.go-122-	Min        schema.Float   `json:"min"`        // Minimum value in time range
internal/metricstoreclient/cc-metric-store.go-123-	Max        schema.Float   `json:"max"`        // Maximum value in time range
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-124-}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-125-