   130→	return &CCMetricStore{
   131→		url:           url,
   132→		queryEndpoint: fmt.Sprintf("%s/api/query", url),
   133→		jwt:           token,
   134→		client: http.Client{
   135→			Timeout: 10 * time.Second,
   136→		},
   137→		topologyCache: make(map[string]*schema.Topology),
   138→	}
   139→}
   140→
   141→// doRequest executes an HTTP POST request to the cc-metric-store query API.
   142→// It handles JSON encoding/decoding, authentication, and API versioning.
   143→// The request body is automatically closed to prevent resource leaks.
   144→func (ccms *CCMetricStore) doRequest(
   145→	ctx context.Context,
   146→	body *APIQueryRequest,
   147→) (*APIQueryResponse, error) {
   148→	buf := &bytes.Buffer{}
   149→	if err := json.NewEncoder(buf).Encode(body); err != nil {
   150→		cclog.Errorf("Error while encoding request body: %s", err.Error())
   151→		return nil, err
   152→	}
   153→
   154→	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf)
   155→	if err != nil {
   156→		cclog.Errorf("Error while building request body: %s", err.Error())
   157→		return nil, err
   158→	}
   159→	if ccms.jwt != "" {
   160→		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   161→	}
   162→
   163→	// versioning the cc-metric-store query API.
   164→	// v2 = data with resampling
   165→	// v1 = data without resampling
   166→	q := req.URL.Query()
   167→	q.Add("version", "v2")
   168→	req.URL.RawQuery = q.Encode()
   169→
   170→	res, err := ccms.client.Do(req)
   171→	if err != nil {
   172→		cclog.Errorf("Error while performing request: %s", err.Error())
   173→		return nil, err
   174→	}
   175→	defer res.Body.Close()
   176→
   177→	if res.StatusCode != http.StatusOK {
   178→		return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status)
   179→	}
   180→
   181→	var resBody APIQueryResponse
   182→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
   183→		cclog.Errorf("Error while decoding result body: %s", err.Error())
   184→		return nil, err
   185→	}
   186→
   187→	return &resBody, nil
   188→}
   189→
   190→// getTopology returns the topology for a given cluster and subcluster, caching it if not already present
   191→func (ccms *CCMetricStore) getTopology(cluster, subCluster string) (*schema.Topology, error) {
   192→	cacheKey := fmt.Sprintf("%s:%s", cluster, subCluster)
   193→	if topology, ok := ccms.topologyCache[cacheKey]; ok {
   194→		return topology, nil
   195→	}
   196→
   197→	subcluster, err := archive.GetSubCluster(cluster, subCluster)
   198→	if err != nil {
   199→		return nil, err
   200→	}
   201→
   202→	ccms.topologyCache[cacheKey] = &subcluster.Topology
   203→	return &subcluster.Topology, nil
   204→}
   205→
   206→// getTopologyByNode returns the topology for a given cluster and node, caching it if not already present
   207→func (ccms *CCMetricStore) getTopologyByNode(cluster, node string) (*schema.Topology, error) {
   208→	subCluster, err := archive.GetSubClusterByNode(cluster, node)
   209→	if err != nil {
   210→		return nil, err
   211→	}
   212→
   213→	return ccms.getTopology(cluster, subCluster)
   214→}
   215→
   216→// LoadData retrieves time series data and statistics for the specified job and metrics.
   217→// It queries data for the job's time range and resources, handling scope transformations automatically.
   218→//
   219→// Parameters:
   220→//   - job: Job metadata including cluster, time range, and allocated resources
   221→//   - metrics: List of metric names to retrieve
   222→//   - scopes: Requested metric scopes (node, socket, core, etc.)
   223→//   - ctx: Context for cancellation and timeouts
   224→//   - resolution: Data resolution in seconds (0 for native resolution)
   225→//
   226→// Returns JobData organized as: metric -> scope -> series list.
   227→// Supports partial errors: returns available data even if some queries fail.
   228→func (ccms *CCMetricStore) LoadData(
   229→	job *schema.Job,
   230→	metrics []string,
   231→	scopes []schema.MetricScope,
   232→	ctx context.Context,
   233→	resolution int,
   234→) (schema.JobData, error) {
   235→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
   236→	if err != nil {
   237→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   238→		return nil, err
   239→	}
   240→
   241→	// Verify assignment is correct - log any inconsistencies for debugging
   242→	if len(queries) != len(assignedScope) {
   243→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
   244→			len(queries), len(assignedScope))
   245→	}
   246→
   247→	req := APIQueryRequest{
   248→		Cluster:   job.Cluster,
   249→		From:      job.StartTime,
   250→		To:        job.StartTime + int64(job.Duration),
   251→		Queries:   queries,
   252→		WithStats: true,
   253→		WithData:  true,
   254→	}
   255→
   256→	resBody, err := ccms.doRequest(ctx, &req)
   257→	if err != nil {
   258→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   259→		return nil, err
   260→	}
   261→
   262→	var errors []string
   263→	jobData := make(schema.JobData)
   264→
   265→	// Add safety check for potential index out of range errors
   266→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   267→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   268→			len(req.Queries), len(resBody.Results), len(assignedScope))
   269→		if len(resBody.Results) > len(req.Queries) {
   270→			resBody.Results = resBody.Results[:len(req.Queries)]
   271→		}
   272→		if len(assignedScope) > len(req.Queries) {
   273→			assignedScope = assignedScope[:len(req.Queries)]
   274→		}
   275→	}
   276→
   277→	for i, row := range resBody.Results {
   278→		// Safety check to prevent index out of range errors
   279→		if i >= len(req.Queries) || i >= len(assignedScope) {
   280→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   281→				i, len(req.Queries), len(assignedScope))
   282→			continue
   283→		}
   284→
   285→		query := req.Queries[i]
   286→		metric := query.Metric
   287→		scope := assignedScope[i]
   288→		mc := archive.GetMetricConfig(job.Cluster, metric)
   289→
   290→		if mc == nil {
   291→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   292→			continue
   293→		}
   294→
   295→		if _, ok := jobData[metric]; !ok {
   296→			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
   297→		}
   298→
   299→		res := mc.Timestep
   300→		if len(row) > 0 {
   301→			res = row[0].Resolution
   302→		}
   303→
   304→		jobMetric, ok := jobData[metric][scope]
   305→		if !ok {
   306→			jobMetric = &schema.JobMetric{
   307→				Unit:     mc.Unit,
   308→				Timestep: res,
   309→				Series:   make([]schema.Series, 0),
   310→			}
   311→			jobData[metric][scope] = jobMetric
   312→		}
   313→
   314→		for ndx, res := range row {
   315→			if res.Error != nil {
   316→				/* Build list for "partial errors", if any */
   317→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   318→				continue
   319→			}
   320→
   321→			id := (*string)(nil)
   322→			if query.Type != nil {
   323→				// Check if ndx is within the bounds of TypeIds slice
   324→				if ndx < len(query.TypeIds) {
   325→					id = new(string)
   326→					*id = query.TypeIds[ndx]
   327→				} else {
   328→					// Log the error but continue processing
   329→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   330→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   331→				}
   332→			}
   333→
   334→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   335→
   336→			jobMetric.Series = append(jobMetric.Series, schema.Series{
   337→				Hostname: query.Hostname,
   338→				ID:       id,
   339→				Statistics: schema.MetricStatistics{
   340→					Avg: float64(res.Avg),
   341→					Min: float64(res.Min),
   342→					Max: float64(res.Max),
   343→				},
   344→				Data: res.Data,
   345→			})
   346→		}
   347→
   348→		// So that one can later check len(jobData):
   349→		if len(jobMetric.Series) == 0 {
   350→			delete(jobData[metric], scope)
   351→			if len(jobData[metric]) == 0 {
   352→				delete(jobData, metric)
   353→			}
   354→		}
   355→	}
   356→
   357→	if len(errors) != 0 {
   358→		/* Returns list for "partial errors" */
   359→		return jobData, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   360→	}
   361→	return jobData, nil
   362→}
   363→
   364→// LoadStats retrieves min/avg/max statistics for job metrics at node scope.
   365→// This is faster than LoadData when only statistical summaries are needed (no time series data).
   366→//
   367→// Returns statistics organized as: metric -> hostname -> statistics.
   368→func (ccms *CCMetricStore) LoadStats(
   369→	job *schema.Job,
   370→	metrics []string,
   371→	ctx context.Context,
   372→) (map[string]map[string]schema.MetricStatistics, error) {
   373→	queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization?
   374→	if err != nil {
   375→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
   376→		return nil, err
   377→	}
   378→
   379→	req := APIQueryRequest{
   380→		Cluster:   job.Cluster,
   381→		From:      job.StartTime,
   382→		To:        job.StartTime + int64(job.Duration),
   383→		Queries:   queries,
   384→		WithStats: true,
   385→		WithData:  false,
   386→	}
   387→
   388→	resBody, err := ccms.doRequest(ctx, &req)
   389→	if err != nil {
   390→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   391→		return nil, err
   392→	}
   393→
   394→	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
   395→	for i, res := range resBody.Results {
   396→		if len(res) == 0 {
   397→			// No Data Found For Metric, Logged in FetchData to Warn
   398→			continue
   399→		}
   400→		query := req.Queries[i]
   401→		metric := query.Metric
   402→		data := res[0]
   403→		if data.Error != nil {
   404→			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
   405→			continue
   406→		}
   407→
   408→		metricdata, ok := stats[metric]
   409→		if !ok {
   410→			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
   411→			stats[metric] = metricdata
   412→		}
   413→
   414→		if hasNaNStats(data.Avg, data.Min, data.Max) {
   415→			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
   416→			continue
   417→		}
   418→
   419→		metricdata[query.Hostname] = schema.MetricStatistics{
   420→			Avg: float64(data.Avg),
   421→			Min: float64(data.Min),
   422→			Max: float64(data.Max),
   423→		}
   424→	}
   425→
   426→	return stats, nil
   427→}
   428→
   429→// LoadScopedStats retrieves statistics for job metrics across multiple scopes.
   430→// Used for the Job-View Statistics Table to display per-scope breakdowns.
   431→//
   432→// Returns statistics organized as: metric -> scope -> list of scoped statistics.
   433→// Each scoped statistic includes hostname, hardware ID (if applicable), and min/avg/max values.
   434→func (ccms *CCMetricStore) LoadScopedStats(
   435→	job *schema.Job,
   436→	metrics []string,
   437→	scopes []schema.MetricScope,
   438→	ctx context.Context,
   439→) (schema.ScopedJobStats, error) {
   440→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
   441→	if err != nil {
   442→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   443→		return nil, err
   444→	}
   445→
   446→	req := APIQueryRequest{
   447→		Cluster:   job.Cluster,
   448→		From:      job.StartTime,
   449→		To:        job.StartTime + int64(job.Duration),
   450→		Queries:   queries,
   451→		WithStats: true,
   452→		WithData:  false,
   453→	}
   454→
   455→	resBody, err := ccms.doRequest(ctx, &req)
   456→	if err != nil {
   457→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   458→		return nil, err
   459→	}
   460→
   461→	var errors []string
   462→	scopedJobStats := make(schema.ScopedJobStats)
   463→
   464→	for i, row := range resBody.Results {
   465→		query := req.Queries[i]
   466→		metric := query.Metric
   467→		scope := assignedScope[i]
   468→
   469→		if _, ok := scopedJobStats[metric]; !ok {
   470→			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
   471→		}
   472→
   473→		if _, ok := scopedJobStats[metric][scope]; !ok {
   474→			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
   475→		}
   476→
   477→		for ndx, res := range row {
   478→			if res.Error != nil {
   479→				/* Build list for "partial errors", if any */
   480→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   481→				continue
   482→			}
   483→
   484→			id := (*string)(nil)
   485→			if query.Type != nil {
   486→				// Check if ndx is within the bounds of TypeIds slice
   487→				if ndx < len(query.TypeIds) {
   488→					id = new(string)
   489→					*id = query.TypeIds[ndx]
   490→				} else {
   491→					// Log the error but continue processing
   492→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   493→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   494→				}
   495→			}
   496→
   497→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   498→
   499→			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
   500→				Hostname: query.Hostname,
   501→				ID:       id,
   502→				Data: &schema.MetricStatistics{
   503→					Avg: float64(res.Avg),
   504→					Min: float64(res.Min),
   505→					Max: float64(res.Max),
   506→				},
   507→			})
   508→		}
   509→
   510→		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
   511→		if len(scopedJobStats[metric][scope]) == 0 {
   512→			delete(scopedJobStats[metric], scope)
   513→			if len(scopedJobStats[metric]) == 0 {
   514→				delete(scopedJobStats, metric)
   515→			}
   516→		}
   517→	}
   518→
   519→	if len(errors) != 0 {
   520→		/* Returns list for "partial errors" */
   521→		return scopedJobStats, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   522→	}
   523→	return scopedJobStats, nil
   524→}
   525→
   526→// LoadNodeData retrieves current metric data for specified nodes in a cluster.
   527→// Used for the Systems-View Node-Overview to display real-time node status.
   528→//
   529→// If nodes is nil, queries all metrics for all nodes in the cluster (bulk query).
   530→// Returns data organized as: hostname -> metric -> list of JobMetric (with time series and stats).
   531→func (ccms *CCMetricStore) LoadNodeData(
   532→	cluster string,
   533→	metrics, nodes []string,
   534→	scopes []schema.MetricScope,
   535→	from, to time.Time,
   536→	ctx context.Context,
   537→) (map[string]map[string][]*schema.JobMetric, error) {
   538→	req := APIQueryRequest{
   539→		Cluster:   cluster,
   540→		From:      from.Unix(),
   541→		To:        to.Unix(),
   542→		WithStats: true,
   543→		WithData:  true,
   544→	}
   545→
   546→	if nodes == nil {
   547→		req.ForAllNodes = append(req.ForAllNodes, metrics...)
   548→	} else {
   549→		for _, node := range nodes {
   550→			for _, metric := range metrics {
   551→				req.Queries = append(req.Queries, APIQuery{
   552→					Hostname:   node,
   553→					Metric:     metric,
   554→					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
   555→				})
   556→			}
   557→		}
   558→	}
   559→
   560→	resBody, err := ccms.doRequest(ctx, &req)
   561→	if err != nil {
   562→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   563→		return nil, err
   564→	}
   565→
   566→	var errors []string
   567→	data := make(map[string]map[string][]*schema.JobMetric)
   568→	for i, res := range resBody.Results {
   569→		if len(res) == 0 {
   570→			// No Data Found For Metric, Logged in FetchData to Warn
   571→			continue
   572→		}
   573→
   574→		var query APIQuery
   575→		if resBody.Queries != nil {
   576→			query = resBody.Queries[i]
   577→		} else {
   578→			query = req.Queries[i]
   579→		}
   580→
   581→		metric := query.Metric
   582→		qdata := res[0]
   583→		if qdata.Error != nil {
   584→			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
   585→		}
   586→
   587→		sanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)
   588→
   589→		hostdata, ok := data[query.Hostname]
   590→		if !ok {
   591→			hostdata = make(map[string][]*schema.JobMetric)
   592→			data[query.Hostname] = hostdata
   593→		}
   594→
   595→		mc := archive.GetMetricConfig(cluster, metric)
   596→		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
   597→			Unit:     mc.Unit,
   598→			Timestep: mc.Timestep,
   599→			Series: []schema.Series{
   600→				{
   601→					Hostname: query.Hostname,
   602→					Data:     qdata.Data,
   603→					Statistics: schema.MetricStatistics{
   604→						Avg: float64(qdata.Avg),
   605→						Min: float64(qdata.Min),
   606→						Max: float64(qdata.Max),
   607→					},
   608→				},
   609→			},
   610→		})
   611→	}
   612→
   613→	if len(errors) != 0 {
   614→		/* Returns list of "partial errors" */
   615→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   616→	}
   617→
   618→	return data, nil
   619→}
   620→
   621→// LoadNodeListData retrieves paginated node metrics for the Systems-View Node-List.
   622→//
   623→// Supports filtering by subcluster and node name pattern. The nodeFilter performs
   624→// substring matching on hostnames.
   625→//
   626→// Returns:
   627→//   - Node data organized as: hostname -> JobData (metric -> scope -> series)
   628→//   - Total node count (before pagination)
   629→//   - HasNextPage flag indicating if more pages are available
   630→//   - Error (may be partial error with some data returned)
   631→func (ccms *CCMetricStore) LoadNodeListData(
   632→	cluster, subCluster string,
   633→	nodes []string,
   634→	metrics []string,
   635→	scopes []schema.MetricScope,
   636→	resolution int,
   637→	from, to time.Time,
   638→	ctx context.Context,
   639→) (map[string]schema.JobData, error) {
   640→	queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
   641→	if err != nil {
   642→		cclog.Errorf("Error while building node queries for Cluster %s, SubCluster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
   643→		return nil, err
   644→	}
   645→
   646→	// Verify assignment is correct - log any inconsistencies for debugging
   647→	if len(queries) != len(assignedScope) {
   648→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
   649→			len(queries), len(assignedScope))
   650→	}
   651→
   652→	req := APIQueryRequest{
   653→		Cluster:   cluster,
   654→		Queries:   queries,
   655→		From:      from.Unix(),
   656→		To:        to.Unix(),
   657→		WithStats: true,
   658→		WithData:  true,
   659→	}
   660→
   661→	resBody, err := ccms.doRequest(ctx, &req)
   662→	if err != nil {
   663→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   664→		return nil, err
   665→	}
   666→
   667→	var errors []string
   668→	data := make(map[string]schema.JobData)
   669→
   670→	// Add safety check for index out of range issues
   671→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   672→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   673→			len(req.Queries), len(resBody.Results), len(assignedScope))
   674→		if len(resBody.Results) > len(req.Queries) {
   675→			resBody.Results = resBody.Results[:len(req.Queries)]
   676→		}
   677→		if len(assignedScope) > len(req.Queries) {
   678→			assignedScope = assignedScope[:len(req.Queries)]
   679→		}
   680→	}
   681→
   682→	for i, row := range resBody.Results {
   683→		// Safety check to prevent index out of range errors
   684→		if i >= len(req.Queries) || i >= len(assignedScope) {
   685→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   686→				i, len(req.Queries), len(assignedScope))
   687→			continue
   688→		}
   689→
   690→		var query APIQuery
   691→		if resBody.Queries != nil {
   692→			if i < len(resBody.Queries) {
   693→				query = resBody.Queries[i]
   694→			} else {
   695→				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
   696→					i, len(resBody.Queries))
   697→				continue
   698→			}
   699→		} else {
   700→			query = req.Queries[i]
   701→		}
   702→
   703→		metric := query.Metric
   704→		scope := assignedScope[i]
   705→		mc := archive.GetMetricConfig(cluster, metric)
   706→		if mc == nil {
   707→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   708→			continue
   709→		}
   710→
   711→		res := mc.Timestep
   712→		if len(row) > 0 {
   713→			res = row[0].Resolution
   714→		}
   715→
   716→		// Init Nested Map Data Structures If Not Found
   717→		hostData, ok := data[query.Hostname]
   718→		if !ok {
   719→			hostData = make(schema.JobData)
   720→			data[query.Hostname] = hostData
   721→		}
   722→
   723→		metricData, ok := hostData[metric]
   724→		if !ok {
   725→			metricData = make(map[schema.MetricScope]*schema.JobMetric)
   726→			data[query.Hostname][metric] = metricData
   727→		}
   728→
   729→		scopeData, ok := metricData[scope]
   730→		if !ok {
   731→			scopeData = &schema.JobMetric{
   732→				Unit:     mc.Unit,
   733→				Timestep: res,
   734→				Series:   make([]schema.Series, 0),
   735→			}
   736→			data[query.Hostname][metric][scope] = scopeData
   737→		}
   738→
   739→		for ndx, res := range row {
   740→			if res.Error != nil {
   741→				/* Build list for "partial errors", if any */
   742→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   743→				continue
   744→			}
   745→
   746→			id := (*string)(nil)
   747→			if query.Type != nil {
   748→				// Check if ndx is within the bounds of TypeIds slice
   749→				if ndx < len(query.TypeIds) {
   750→					id = new(string)
   751→					*id = query.TypeIds[ndx]
   752→				} else {
   753→					// Log the error but continue processing
   754→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   755→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   756→				}
   757→			}
   758→
   759→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   760→
   761→			scopeData.Series = append(scopeData.Series, schema.Series{
   762→				Hostname: query.Hostname,
   763→				ID:       id,
   764→				Statistics: schema.MetricStatistics{
   765→					Avg: float64(res.Avg),
   766→					Min: float64(res.Min),
   767→					Max: float64(res.Max),
   768→				},
   769→				Data: res.Data,
   770→			})
   771→		}
   772→	}
   773→
   774→	if len(errors) != 0 {
   775→		/* Returns list of "partial errors" */
   776→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   777→	}
   778→
   779→	return data, nil
   780→}
   781→
   782→// HealthCheck queries the external cc-metric-store's health check endpoint.
   783→// It sends a HealthCheckReq as the request body to /api/healthcheck and
   784→// returns the per-node health check results.
   785→func (ccms *CCMetricStore) HealthCheck(cluster string,
   786→	nodes []string, metrics []string,
   787→) (map[string]metricstore.HealthCheckResult, error) {
   788→	req := metricstore.HealthCheckReq{
   789→		Cluster:     cluster,
   790→		Nodes:       nodes,
   791→		MetricNames: metrics,
   792→	}
   793→
   794→	buf := &bytes.Buffer{}
   795→	if err := json.NewEncoder(buf).Encode(req); err != nil {
   796→		cclog.Errorf("Error while encoding health check request body: %s", err.Error())
   797→		return nil, err
   798→	}
   799→
   800→	endpoint := fmt.Sprintf("%s/api/healthcheck", ccms.url)
   801→	httpReq, err := http.NewRequest(http.MethodGet, endpoint, buf)
   802→	if err != nil {
   803→		cclog.Errorf("Error while building health check request: %s", err.Error())
   804→		return nil, err
   805→	}
   806→	if ccms.jwt != "" {
   807→		httpReq.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   808→	}
   809→
   810→	res, err := ccms.client.Do(httpReq)
   811→	if err != nil {
   812→		cclog.Errorf("Error while performing health check request: %s", err.Error())
   813→		return nil, err
   814→	}
   815→	defer res.Body.Close()
   816→
   817→	if res.StatusCode != http.StatusOK {
   818→		return nil, fmt.Errorf("'%s': HTTP Status: %s", endpoint, res.Status)
   819→	}
   820→
   821→	var results map[string]metricstore.HealthCheckResult
   822→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&results); err != nil {
   823→		cclog.Errorf("Error while decoding health check response: %s", err.Error())
   824→		return nil, err
   825→	}
   826→
   827→	return results, nil
   828→}
   829→
   830→// sanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
   831→// Regular float64 values cannot be JSONed when NaN.
   832→func sanitizeStats(avg, min, max *schema.Float) {
   833→	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
   834→		*avg = schema.Float(0)
   835→		*min = schema.Float(0)
   836→		*max = schema.Float(0)
   837→	}
   838→}
   839→
   840→// hasNaNStats returns true if any of the statistics contain NaN values.
   841→func hasNaNStats(avg, min, max schema.Float) bool {
   842→	return avg.IsNaN() || min.IsNaN() || max.IsNaN()
   843→}
   844→

	return &CCMetricStore{
		url:           url,
		queryEndpoint: fmt.Sprintf("%s/api/query", url),
		jwt:           token,
		client: http.Client{
			Timeout: 10 * time.Second,
		},
		topologyCache: make(map[string]*schema.Topology),
	}
}

// doRequest executes an HTTP POST request to the cc-metric-store query API.
// It handles JSON encoding/decoding, authentication, and API versioning.
// The request body is automatically closed to prevent resource leaks.
func (ccms *CCMetricStore) doRequest(
	ctx context.Context,
	body *APIQueryRequest,
) (*APIQueryResponse, error) {
	buf := &bytes.Buffer{}
	if err := json.NewEncoder(buf).Encode(body); err != nil {
		cclog.Errorf("Error while encoding request body: %s", err.Error())
		return nil, err
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf)
	if err != nil {
		cclog.Errorf("Error while building request body: %s", err.Error())
		return nil, err
	}
	if ccms.jwt != "" {
		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
	}

	// versioning the cc-metric-store query API.
	// v2 = data with resampling
	// v1 = data without resampling
	q := req.URL.Query()
	q.Add("version", "v2")
	req.URL.RawQuery = q.Encode()

	res, err := ccms.client.Do(req)
	if err != nil {
		cclog.Errorf("Error while performing request: %s", err.Error())
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status)
	}

	var resBody APIQueryResponse
	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
		cclog.Errorf("Error while decoding result body: %s", err.Error())
		return nil, err
	}

	return &resBody, nil
}

// getTopology returns the topology for a given cluster and subcluster, caching it if not already present
func (ccms *CCMetricStore) getTopology(cluster, subCluster string) (*schema.Topology, error) {
	cacheKey := fmt.Sprintf("%s:%s", cluster, subCluster)
	if topology, ok := ccms.topologyCache[cacheKey]; ok {
		return topology, nil
	}

	subcluster, err := archive.GetSubCluster(cluster, subCluster)
	if err != nil {
		return nil, err
	}

	ccms.topologyCache[cacheKey] = &subcluster.Topology
	return &subcluster.Topology, nil
}

// getTopologyByNode returns the topology for a given cluster and node, caching it if not already present
func (ccms *CCMetricStore) getTopologyByNode(cluster, node string) (*schema.Topology, error) {
	subCluster, err := archive.GetSubClusterByNode(cluster, node)
	if err != nil {
		return nil, err
	}

	return ccms.getTopology(cluster, subCluster)
}

// LoadData retrieves time series data and statistics for the specified job and metrics.
// It queries data for the job's time range and resources, handling scope transformations automatically.
//
// Parameters:
//   - job: Job metadata including cluster, time range, and allocated resources
//   - metrics: List of metric names to retrieve
//   - scopes: Requested metric scopes (node, socket, core, etc.)
//   - ctx: Context for cancellation and timeouts
//   - resolution: Data resolution in seconds (0 for native resolution)
//
// Returns JobData organized as: metric -> scope -> series list.
// Supports partial errors: returns available data even if some queries fail.
func (ccms *CCMetricStore) LoadData(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
	resolution int,
) (schema.JobData, error) {
	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  true,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	var errors []string
	jobData := make(schema.JobData)

	// Add safety check for potential index out of range errors
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		// Safety check to prevent index out of range errors
		if i >= len(req.Queries) || i >= len(assignedScope) {
			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
				i, len(req.Queries), len(assignedScope))
			continue
		}

		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(job.Cluster, metric)

		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
			continue
		}

		if _, ok := jobData[metric]; !ok {
			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = row[0].Resolution
		}

		jobMetric, ok := jobData[metric][scope]
		if !ok {
			jobMetric = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			jobData[metric][scope] = jobMetric
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			sanitizeStats(&res.Avg, &res.Min, &res.Max)

			jobMetric.Series = append(jobMetric.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}

		// So that one can later check len(jobData):
		if len(jobMetric.Series) == 0 {
			delete(jobData[metric], scope)
			if len(jobData[metric]) == 0 {
				delete(jobData, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return jobData, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return jobData, nil
}

// LoadStats retrieves min/avg/max statistics for job metrics at node scope.
// This is faster than LoadData when only statistical summaries are needed (no time series data).
//
// Returns statistics organized as: metric -> hostname -> statistics.
func (ccms *CCMetricStore) LoadStats(
	job *schema.Job,
	metrics []string,
	ctx context.Context,
) (map[string]map[string]schema.MetricStatistics, error) {
	queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization?
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
	for i, res := range resBody.Results {
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}
		query := req.Queries[i]
		metric := query.Metric
		data := res[0]
		if data.Error != nil {
			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
			continue
		}

		metricdata, ok := stats[metric]
		if !ok {
			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
			stats[metric] = metricdata
		}

		if hasNaNStats(data.Avg, data.Min, data.Max) {
			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
			continue
		}

		metricdata[query.Hostname] = schema.MetricStatistics{
			Avg: float64(data.Avg),
			Min: float64(data.Min),
			Max: float64(data.Max),
		}
	}

	return stats, nil
}

// LoadScopedStats retrieves statistics for job metrics across multiple scopes.
// Used for the Job-View Statistics Table to display per-scope breakdowns.
//
// Returns statistics organized as: metric -> scope -> list of scoped statistics.
// Each scoped statistic includes hostname, hardware ID (if applicable), and min/avg/max values.
func (ccms *CCMetricStore) LoadScopedStats(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
) (schema.ScopedJobStats, error) {
	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	var errors []string
	scopedJobStats := make(schema.ScopedJobStats)

	for i, row := range resBody.Results {
		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]

		if _, ok := scopedJobStats[metric]; !ok {
			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
		}

		if _, ok := scopedJobStats[metric][scope]; !ok {
			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			sanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
				Hostname: query.Hostname,
				ID:       id,
				Data: &schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
			})
		}

		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
		if len(scopedJobStats[metric][scope]) == 0 {
			delete(scopedJobStats[metric], scope)
			if len(scopedJobStats[metric]) == 0 {
				delete(scopedJobStats, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return scopedJobStats, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return scopedJobStats, nil
}

// LoadNodeData retrieves current metric data for specified nodes in a cluster.
// Used for the Systems-View Node-Overview to display real-time node status.
//
// If nodes is nil, queries all metrics for all nodes in the cluster (bulk query).
// Returns data organized as: hostname -> metric -> list of JobMetric (with time series and stats).
func (ccms *CCMetricStore) LoadNodeData(
	cluster string,
	metrics, nodes []string,
	scopes []schema.MetricScope,
	from, to time.Time,
	ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
	req := APIQueryRequest{
		Cluster:   cluster,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	if nodes == nil {
		req.ForAllNodes = append(req.ForAllNodes, metrics...)
	} else {
		for _, node := range nodes {
			for _, metric := range metrics {
				req.Queries = append(req.Queries, APIQuery{
					Hostname:   node,
					Metric:     metric,
					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
				})
			}
		}
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]map[string][]*schema.JobMetric)
	for i, res := range resBody.Results {
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}

		var query APIQuery
		if resBody.Queries != nil {
			query = resBody.Queries[i]
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		qdata := res[0]
		if qdata.Error != nil {
			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
		}

		sanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)

		hostdata, ok := data[query.Hostname]
		if !ok {
			hostdata = make(map[string][]*schema.JobMetric)
			data[query.Hostname] = hostdata
		}

		mc := archive.GetMetricConfig(cluster, metric)
		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
			Unit:     mc.Unit,
			Timestep: mc.Timestep,
			Series: []schema.Series{
				{
					Hostname: query.Hostname,
					Data:     qdata.Data,
					Statistics: schema.MetricStatistics{
						Avg: float64(qdata.Avg),
						Min: float64(qdata.Min),
						Max: float64(qdata.Max),
					},
				},
			},
		})
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// LoadNodeListData retrieves paginated node metrics for the Systems-View Node-List.
//
// Supports filtering by subcluster and node name pattern. The nodeFilter performs
// substring matching on hostnames.
//
// Returns:
//   - Node data organized as: hostname -> JobData (metric -> scope -> series)
//   - Total node count (before pagination)
//   - HasNextPage flag indicating if more pages are available
//   - Error (may be partial error with some data returned)
func (ccms *CCMetricStore) LoadNodeListData(
	cluster, subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	from, to time.Time,
	ctx context.Context,
) (map[string]schema.JobData, error) {
	queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
	if err != nil {
		cclog.Errorf("Error while building node queries for Cluster %s, SubCluster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:   cluster,
		Queries:   queries,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	resBody, err := ccms.doRequest(ctx, &req)
	if err != nil {
		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]schema.JobData)

	// Add safety check for index out of range issues
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		// Safety check to prevent index out of range errors
		if i >= len(req.Queries) || i >= len(assignedScope) {
			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
				i, len(req.Queries), len(assignedScope))
			continue
		}

		var query APIQuery
		if resBody.Queries != nil {
			if i < len(resBody.Queries) {
				query = resBody.Queries[i]
			} else {
				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
					i, len(resBody.Queries))
				continue
			}
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
			continue
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = row[0].Resolution
		}

		// Init Nested Map Data Structures If Not Found
		hostData, ok := data[query.Hostname]
		if !ok {
			hostData = make(schema.JobData)
			data[query.Hostname] = hostData
		}

		metricData, ok := hostData[metric]
		if !ok {
			metricData = make(map[schema.MetricScope]*schema.JobMetric)
			data[query.Hostname][metric] = metricData
		}

		scopeData, ok := metricData[scope]
		if !ok {
			scopeData = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			data[query.Hostname][metric][scope] = scopeData
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := (*string)(nil)
			if query.Type != nil {
				// Check if ndx is within the bounds of TypeIds slice
				if ndx < len(query.TypeIds) {
					id = new(string)
					*id = query.TypeIds[ndx]
				} else {
					// Log the error but continue processing
					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
						ndx, len(query.TypeIds), query.Metric, query.Hostname)
				}
			}

			sanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopeData.Series = append(scopeData.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// HealthCheck queries the external cc-metric-store's health check endpoint.
// It sends a HealthCheckReq as the request body to /api/healthcheck and
// returns the per-node health check results.
func (ccms *CCMetricStore) HealthCheck(cluster string,
	nodes []string, metrics []string,
) (map[string]metricstore.HealthCheckResult, error) {
	req := metricstore.HealthCheckReq{
		Cluster:     cluster,
		Nodes:       nodes,
		MetricNames: metrics,
	}

	buf := &bytes.Buffer{}
	if err := json.NewEncoder(buf).Encode(req); err != nil {
		cclog.Errorf("Error while encoding health check request body: %s", err.Error())
		return nil, err
	}

	endpoint := fmt.Sprintf("%s/api/healthcheck", ccms.url)
	httpReq, err := http.NewRequest(http.MethodGet, endpoint, buf)
	if err != nil {
		cclog.Errorf("Error while building health check request: %s", err.Error())
		return nil, err
	}
	if ccms.jwt != "" {
		httpReq.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
	}

	res, err := ccms.client.Do(httpReq)
	if err != nil {
		cclog.Errorf("Error while performing health check request: %s", err.Error())
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("'%s': HTTP Status: %s", endpoint, res.Status)
	}

	var results map[string]metricstore.HealthCheckResult
	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&results); err != nil {
		cclog.Errorf("Error while decoding health check response: %s", err.Error())
		return nil, err
	}

	return results, nil
}

// sanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
// Regular float64 values cannot be JSONed when NaN.
func sanitizeStats(avg, min, max *schema.Float) {
	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
		*avg = schema.Float(0)
		*min = schema.Float(0)
		*max = schema.Float(0)
	}
}

// hasNaNStats returns true if any of the statistics contain NaN values.
func hasNaNStats(avg, min, max schema.Float) bool {
	return avg.IsNaN() || min.IsNaN() || max.IsNaN()
}


Now I have a comprehensive understanding. Let me design the plan.


Agent

Design consolidation plan

Prompt:
I need to design a plan to synchronize and consolidate the query-building logic between two packages that implement the same MetricDataRepository interface:

## Context

1. **`pkg/metricstore/query.go`** (InternalMetricStore) — in-memory metric store
2. **`internal/metricstoreclient/cc-metric-store-queries.go`** (CCMetricStore) — HTTP client to external cc-metric-store

Both implement the `MetricDataRepository` interface from `internal/metricdispatch/metricdata.go`.

## Key Differences Found

### buildQueries differences:
- **pkg/metricstore**: Inline scope transformation (long chain of if-statements in the inner loop). Uses `map[schema.MetricScope]bool` for dedup. Resolution is `int64`.
- **metricstoreclient**: Extracted `buildScopeQueries()` helper function. Uses labeled loop + `[]schema.MetricScope` slice for dedup. Resolution is `int`. Has topology caching via `getTopology()`.

### buildScopeQueries (only in metricstoreclient):
- Clean standalone function handling all scope transformations
- Returns empty slices for expected exceptions, nil for unhandled cases
- Includes **MemoryDomain → Socket** transformation (MISSING from pkg/metricstore)

### buildNodeQueries differences:
- **pkg/metricstore**: Gets topology inline per node. Uses `map[schema.MetricScope]bool` dedup. Has inline scope transformation.
- **metricstoreclient**: Uses `getTopology()`/`getTopologyByNode()` with caching. Uses labeled loop dedup. Delegates to `buildScopeQueries()`.

### APIQuery type differences:
- **pkg/metricstore**: `Resolution int64`, has `ScaleFactor schema.Float`
- **metricstoreclient**: `Resolution int`, no ScaleFactor

### sanitizeStats differences:
- **pkg/metricstore**: `sanitizeStats(data *APIMetricData)` — zeros each independently
- **metricstoreclient**: `sanitizeStats(avg, min, max *schema.Float)` — zeros all if any is NaN. Also has `hasNaNStats()` helper.

### LoadData/LoadStats/LoadScopedStats/LoadNodeData/LoadNodeListData:
- Nearly identical result-processing logic
- Different data fetching: `FetchData(req)` vs `ccms.doRequest(ctx, &req)`
- Internal store has `TestLoadDataCallback` test hook
- External client has topology caching

### Missing from pkg/metricstore:
- **MemoryDomain → Socket** scope transformation (present in metricstoreclient)

### Missing from metricstoreclient:
- Nothing significant missing from metricstoreclient

## Consolidation Approach

The user said metricstoreclient is the cleaner implementation. I agree — `buildScopeQueries()` as a standalone function is much cleaner than the inline if-chain.

Design a plan to:
1. Extract `buildScopeQueries()` as a shared helper (possibly in a shared package or as a package-level function in metricstore)
2. Synchronize the scope transformations (add MemoryDomain → Socket to internal)
3. Evaluate if result-processing logic can be shared
4. Consider type differences (int vs int64 Resolution)

Key constraints:
- The two packages have separate APIQuery types (different Resolution types, internal has ScaleFactor)
- The two packages use different data fetching mechanisms
- Must not break the MetricDataRepository interface contract
- Keep changes minimal and focused

Provide a detailed implementation plan.


I need to design a plan to synchronize and consolidate the query-building logic between two packages that implement the same MetricDataRepository interface:

## Context

1. **`pkg/metricstore/query.go`** (InternalMetricStore) — in-memory metric store
2. **`internal/metricstoreclient/cc-metric-store-queries.go`** (CCMetricStore) — HTTP client to external cc-metric-store

Both implement the `MetricDataRepository` interface from `internal/metricdispatch/metricdata.go`.

## Key Differences Found

### buildQueries differences:
- **pkg/metricstore**: Inline scope transformation (long chain of if-statements in the inner loop). Uses `map[schema.MetricScope]bool` for dedup. Resolution is `int64`.
- **metricstoreclient**: Extracted `buildScopeQueries()` helper function. Uses labeled loop + `[]schema.MetricScope` slice for dedup. Resolution is `int`. Has topology caching via `getTopology()`.

### buildScopeQueries (only in metricstoreclient):
- Clean standalone function handling all scope transformations
- Returns empty slices for expected exceptions, nil for unhandled cases
- Includes **MemoryDomain → Socket** transformation (MISSING from pkg/metricstore)

### buildNodeQueries differences:
- **pkg/metricstore**: Gets topology inline per node. Uses `map[schema.MetricScope]bool` dedup. Has inline scope transformation.
- **metricstoreclient**: Uses `getTopology()`/`getTopologyByNode()` with caching. Uses labeled loop dedup. Delegates to `buildScopeQueries()`.

### APIQuery type differences:
- **pkg/metricstore**: `Resolution int64`, has `ScaleFactor schema.Float`
- **metricstoreclient**: `Resolution int`, no ScaleFactor

### sanitizeStats differences:
- **pkg/metricstore**: `sanitizeStats(data *APIMetricData)` — zeros each independently
- **metricstoreclient**: `sanitizeStats(avg, min, max *schema.Float)` — zeros all if any is NaN. Also has `hasNaNStats()` helper.

### LoadData/LoadStats/LoadScopedStats/LoadNodeData/LoadNodeListData:
- Nearly identical result-processing logic
- Different data fetching: `FetchData(req)` vs `ccms.doRequest(ctx, &req)`
- Internal store has `TestLoadDataCallback` test hook
- External client has topology caching

### Missing from pkg/metricstore:
- **MemoryDomain → Socket** scope transformation (present in metricstoreclient)

### Missing from metricstoreclient:
- Nothing significant missing from metricstoreclient

## Consolidation Approach

The user said metricstoreclient is the cleaner implementation. I agree — `buildScopeQueries()` as a standalone function is much cleaner than the inline if-chain.

Design a plan to:
1. Extract `buildScopeQueries()` as a shared helper (possibly in a shared package or as a package-level function in metricstore)
2. Synchronize the scope transformations (add MemoryDomain → Socket to internal)
3. Evaluate if result-processing logic can be shared
4. Consider type differences (int vs int64 Resolution)

Key constraints:
- The two packages have separate APIQuery types (different Resolution types, internal has ScaleFactor)
- The two packages use different data fetching mechanisms
- Must not break the MetricDataRepository interface contract
- Keep changes minimal and focused

Provide a detailed implementation plan.

Prompt:
I need to design a plan to synchronize and consolidate the query-building logic between two packages that implement the same MetricDataRepository interface:

## Context

1. **`pkg/metricstore/query.go`** (InternalMetricStore) — in-memory metric store
2. **`internal/metricstoreclient/cc-metric-store-queries.go`** (CCMetricStore) — HTTP client to external cc-metric-store

Both implement the `MetricDataRepository` interface from `internal/metricdispatch/metricdata.go`.

## Key Differences Found

### buildQueries differences:
- **pkg/metricstore**: Inline scope transformation (long chain of if-statements in the inner loop). Uses `map[schema.MetricScope]bool` for dedup. Resolution is `int64`.
- **metricstoreclient**: Extracted `buildScopeQueries()` helper function. Uses labeled loop + `[]schema.MetricScope` slice for dedup. Resolution is `int`. Has topology caching via `getTopology()`.

### buildScopeQueries (only in metricstoreclient):
- Clean standalone function handling all scope transformations
- Returns empty slices for expected exceptions, nil for unhandled cases
- Includes **MemoryDomain → Socket** transformation (MISSING from pkg/metricstore)

### buildNodeQueries differences:
- **pkg/metricstore**: Gets topology inline per node. Uses `map[schema.MetricScope]bool` dedup. Has inline scope transformation.
- **metricstoreclient**: Uses `getTopology()`/`getTopologyByNode()` with caching. Uses labeled loop dedup. Delegates to `buildScopeQueries()`.

### APIQuery type differences:
- **pkg/metricstore**: `Resolution int64`, has `ScaleFactor schema.Float`
- **metricstoreclient**: `Resolution int`, no ScaleFactor

### sanitizeStats differences:
- **pkg/metricstore**: `sanitizeStats(data *APIMetricData)` — zeros each independently
- **metricstoreclient**: `sanitizeStats(avg, min, max *schema.Float)` — zeros all if any is NaN. Also has `hasNaNStats()` helper.

### LoadData/LoadStats/LoadScopedStats/LoadNodeData/LoadNodeListData:
- Nearly identical result-processing logic
- Different data fetching: `FetchData(req)` vs `ccms.doRequest(ctx, &req)`
- Internal store has `TestLoadDataCallback` test hook
- External client has topology caching

### Missing from pkg/metricstore:
- **MemoryDomain → Socket** scope transformation (present in metricstoreclient)

### Missing from metricstoreclient:
- Nothing significant missing from metricstoreclient

## Consolidation Approach

The user said metricstoreclient is the cleaner implementation. I agree — `buildScopeQueries()` as a standalone function is much cleaner than the inline if-chain.

Design a plan to:
1. Extract `buildScopeQueries()` as a shared helper (possibly in a shared package or as a package-level function in metricstore)
2. Synchronize the scope transformations (add MemoryDomain → Socket to internal)
3. Evaluate if result-processing logic can be shared
4. Consider type differences (int vs int64 Resolution)

Key constraints:
- The two packages have separate APIQuery types (different Resolution types, internal has ScaleFactor)
- The two packages use different data fetching mechanisms
- Must not break the MetricDataRepository interface contract
- Keep changes minimal and focused

Provide a detailed implementation plan.


<persisted-output>
Output too large (50.6KB). Full output saved to: /Users/jan/.claude/projects/-Users-jan-prg-ClusterCockpit-cc-backend/eec1fc87-d8f6-42f6-b296-d7a2f79af002/tool-results/toolu_01SJwjDjjHUssBPJtycj9NVd.txt

Preview (first 2KB):
     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements high-level query functions for loading job metric data
     7→// with automatic scope transformation and aggregation.
     8→//
     9→// Key Concepts:
    10→//
    11→// Metric Scopes: Metrics are collected at different granularities (native scope):
    12→//   - HWThread: Per hardware thread
    13→//   - Core: Per CPU core
    14→//   - Socket: Per CPU socket
    15→//   - MemoryDomain: Per memory domain (NUMA)
    16→//   - Accelerator: Per GPU/accelerator
    17→//   - Node: Per compute node
    18→//
    19→// Scope Transformation: The buildQueries functions transform between native scope
    20→// and requested scope by:
    21→//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
    22→//   - Rejecting requests for finer granularity than available
    23→//   - Handling special cases (e.g., Accelerator metrics)
    24→//
    25→// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
    26→// based on cluster topology and job resources.
    27→package metricstore
    28→
    29→import (
    30→	"context"
    31→	"fmt"
    32→	"strconv"
    33→	"strings"
    34→	"time"
    35→
    36→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    37→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    38→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    39→)
    40→
    41→type InternalMetricStore struct{}
    42→
    43→var MetricStoreHandle *InternalMetricStore
    44→
    45→// HealthCheck delegates to the internal MemoryStore's HealthCheck.
    46→func (ccms *InternalMetricStore) HealthCheck(cluster string,
    47→	nodes []string, metrics []string,
    48→) (map[string]HealthCheckResult, error) {
...
</persisted-output>


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstoreclient - Query Building
     7→//
     8→// This file contains the query construction and scope transformation logic for cc-metric-store queries.
     9→// It handles the complex mapping between requested metric scopes and native hardware topology,
    10→// automatically aggregating or filtering metrics as needed.
    11→//
    12→// # Scope Transformations
    13→//
    14→// The buildScopeQueries function implements the core scope transformation algorithm.
    15→// It handles 25+ different transformation cases, mapping between:
    16→//   - Accelerator (GPU) scope
    17→//   - HWThread (hardware thread/SMT) scope
    18→//   - Core (CPU core) scope
    19→//   - Socket (CPU package) scope
    20→//   - MemoryDomain (NUMA domain) scope
    21→//   - Node (full system) scope
    22→//
    23→// Transformations follow these rules:
    24→//   - Same scope: Return data as-is (e.g., Core → Core)
    25→//   - Coarser scope: Aggregate data (e.g., Core → Socket with Aggregate=true)
    26→//   - Finer scope: Error - cannot increase granularity
    27→//
    28→// # Query Building
    29→//
    30→// buildQueries and buildNodeQueries are the main entry points, handling job-specific
    31→// and node-specific query construction respectively. They:
    32→//   - Validate metric configurations
    33→//   - Handle subcluster-specific metric filtering
    34→//   - Detect and skip duplicate scope requests
    35→//   - Call buildScopeQueries for each metric/scope/host combination
    36→package metricstoreclient
    37→
    38→import (
    39→	"fmt"
    40→	"strconv"
    41→
    42→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    43→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    44→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    45→)
    46→
    47→// Scope string constants used in API queries.
    48→// Pre-converted to avoid repeated allocations during query building.
    49→var (
    50→	hwthreadString     = string(schema.MetricScopeHWThread)
    51→	coreString         = string(schema.MetricScopeCore)
    52→	memoryDomainString = string(schema.MetricScopeMemoryDomain)
    53→	socketString       = string(schema.MetricScopeSocket)
    54→	acceleratorString  = string(schema.MetricScopeAccelerator)
    55→)
    56→
    57→// buildQueries constructs API queries for job-specific metric data.
    58→// It iterates through metrics, scopes, and job resources to build the complete query set.
    59→//
    60→// The function handles:
    61→//   - Metric configuration validation and subcluster filtering
    62→//   - Scope deduplication to avoid redundant queries
    63→//   - Hardware thread list resolution (job-allocated vs full node)
    64→//   - Delegation to buildScopeQueries for scope transformations
    65→//
    66→// Returns queries and their corresponding assigned scopes (which may differ from requested scopes).
    67→func (ccms *CCMetricStore) buildQueries(
    68→	job *schema.Job,
    69→	metrics []string,
    70→	scopes []schema.MetricScope,
    71→	resolution int,
    72→) ([]APIQuery, []schema.MetricScope, error) {
    73→	// Initialize both slices together
    74→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
    75→	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(job.Resources))
    76→
    77→	topology, err := ccms.getTopology(job.Cluster, job.SubCluster)
    78→	if err != nil {
    79→		cclog.Errorf("could not load cluster %s subCluster %s topology: %s", job.Cluster, job.SubCluster, err.Error())
    80→		return nil, nil, err
    81→	}
    82→
    83→	for _, metric := range metrics {
    84→		remoteName := metric
    85→		mc := archive.GetMetricConfig(job.Cluster, metric)
    86→		if mc == nil {
    87→			cclog.Warnf("metric '%s' is not specified for cluster '%s' - skipping", metric, job.Cluster)
    88→			continue
    89→		}
    90→
    91→		// Skip if metric is removed for subcluster
    92→		if len(mc.SubClusters) != 0 {
    93→			isRemoved := false
    94→			for _, scConfig := range mc.SubClusters {
    95→				if scConfig.Name == job.SubCluster && scConfig.Remove {
    96→					isRemoved = true
    97→					break
    98→				}
    99→			}
   100→			if isRemoved {
   101→				continue
   102→			}
   103→		}
   104→
   105→		// Avoid duplicates...
   106→		handledScopes := make([]schema.MetricScope, 0, 3)
   107→
   108→	scopesLoop:
   109→		for _, requestedScope := range scopes {
   110→			nativeScope := mc.Scope
   111→			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
   112→				continue
   113→			}
   114→
   115→			scope := nativeScope.Max(requestedScope)
   116→			for _, s := range handledScopes {
   117→				if scope == s {
   118→					continue scopesLoop
   119→				}
   120→			}
   121→			handledScopes = append(handledScopes, scope)
   122→
   123→			for _, host := range job.Resources {
   124→				hwthreads := host.HWThreads
   125→				if hwthreads == nil {
   126→					hwthreads = topology.Node
   127→				}
   128→
   129→				// Note: Expected exceptions will return as empty slices -> Continue
   130→				hostQueries, hostScopes := buildScopeQueries(
   131→					nativeScope, requestedScope,
   132→					remoteName, host.Hostname,
   133→					topology, hwthreads, host.Accelerators,
   134→					resolution,
   135→				)
   136→
   137→				// Note: Unexpected errors, such as unhandled cases, will return as nils -> Error
   138→				if hostQueries == nil && hostScopes == nil {
   139→					return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   140→				}
   141→
   142→				queries = append(queries, hostQueries...)
   143→				assignedScope = append(assignedScope, hostScopes...)
   144→			}
   145→		}
   146→	}
   147→
   148→	return queries, assignedScope, nil
   149→}
   150→
   151→// buildNodeQueries constructs API queries for node-specific metric data (Systems View).
   152→// Similar to buildQueries but uses full node topology instead of job-allocated resources.
   153→//
   154→// The function handles:
   155→//   - SubCluster topology resolution (either pre-loaded or per-node lookup)
   156→//   - Full node hardware thread lists (not job-specific subsets)
   157→//   - All accelerators on each node
   158→//   - Metric configuration validation with subcluster filtering
   159→//
   160→// Returns queries and their corresponding assigned scopes.
   161→func (ccms *CCMetricStore) buildNodeQueries(
   162→	cluster string,
   163→	subCluster string,
   164→	nodes []string,
   165→	metrics []string,
   166→	scopes []schema.MetricScope,
   167→	resolution int,
   168→) ([]APIQuery, []schema.MetricScope, error) {
   169→	// Initialize both slices together
   170→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
   171→	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(nodes))
   172→
   173→	for _, metric := range metrics {
   174→		remoteName := metric
   175→		mc := archive.GetMetricConfig(cluster, metric)
   176→		if mc == nil {
   177→			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
   178→			continue
   179→		}
   180→
   181→		// Skip if metric is removed for subcluster
   182→		if mc.SubClusters != nil {
   183→			isRemoved := false
   184→			for _, scConfig := range mc.SubClusters {
   185→				if scConfig.Name == subCluster && scConfig.Remove {
   186→					isRemoved = true
   187→					break
   188→				}
   189→			}
   190→			if isRemoved {
   191→				continue
   192→			}
   193→		}
   194→
   195→		// Avoid duplicates...
   196→		handledScopes := make([]schema.MetricScope, 0, 3)
   197→
   198→	scopesLoop:
   199→		for _, requestedScope := range scopes {
   200→			nativeScope := mc.Scope
   201→
   202→			scope := nativeScope.Max(requestedScope)
   203→			for _, s := range handledScopes {
   204→				if scope == s {
   205→					continue scopesLoop
   206→				}
   207→			}
   208→			handledScopes = append(handledScopes, scope)
   209→
   210→			for _, hostname := range nodes {
   211→				var topology *schema.Topology
   212→				var err error
   213→
   214→				// If no subCluster given, get it by node
   215→				if subCluster == "" {
   216→					topology, err = ccms.getTopologyByNode(cluster, hostname)
   217→				} else {
   218→					topology, err = ccms.getTopology(cluster, subCluster)
   219→				}
   220→
   221→				if err != nil {
   222→					return nil, nil, err
   223→				}
   224→
   225→				// Always full node hwthread id list, no partial queries expected -> Use "topology.Node" directly where applicable
   226→				// Always full accelerator id list, no partial queries expected -> Use "acceleratorIds" directly where applicable
   227→				acceleratorIds := topology.GetAcceleratorIDs()
   228→
   229→				// Moved check here if metric matches hardware specs
   230→				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
   231→					continue scopesLoop
   232→				}
   233→
   234→				nodeQueries, nodeScopes := buildScopeQueries(
   235→					nativeScope, requestedScope,
   236→					remoteName, hostname,
   237→					topology, topology.Node, acceleratorIds,
   238→					resolution,
   239→				)
   240→
   241→				if len(nodeQueries) == 0 && len(nodeScopes) == 0 {
   242→					return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   243→				}
   244→
   245→				queries = append(queries, nodeQueries...)
   246→				assignedScope = append(assignedScope, nodeScopes...)
   247→			}
   248→		}
   249→	}
   250→
   251→	return queries, assignedScope, nil
   252→}
   253→
   254→// buildScopeQueries generates API queries for a given scope transformation.
   255→// It returns a slice of queries and corresponding assigned scopes.
   256→// Some transformations (e.g., HWThread -> Core/Socket) may generate multiple queries.
   257→func buildScopeQueries(
   258→	nativeScope, requestedScope schema.MetricScope,
   259→	metric, hostname string,
   260→	topology *schema.Topology,
   261→	hwthreads []int,
   262→	accelerators []string,
   263→	resolution int,
   264→) ([]APIQuery, []schema.MetricScope) {
   265→	scope := nativeScope.Max(requestedScope)
   266→	queries := []APIQuery{}
   267→	scopes := []schema.MetricScope{}
   268→
   269→	hwthreadsStr := intToStringSlice(hwthreads)
   270→
   271→	// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
   272→	if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
   273→		if scope != schema.MetricScopeAccelerator {
   274→			// Expected Exception -> Continue -> Return Empty Slices
   275→			return queries, scopes
   276→		}
   277→
   278→		queries = append(queries, APIQuery{
   279→			Metric:     metric,
   280→			Hostname:   hostname,
   281→			Aggregate:  false,
   282→			Type:       &acceleratorString,
   283→			TypeIds:    accelerators,
   284→			Resolution: resolution,
   285→		})
   286→		scopes = append(scopes, schema.MetricScopeAccelerator)
   287→		return queries, scopes
   288→	}
   289→
   290→	// Accelerator -> Node
   291→	if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
   292→		if len(accelerators) == 0 {
   293→			// Expected Exception -> Continue -> Return Empty Slices
   294→			return queries, scopes
   295→		}
   296→
   297→		queries = append(queries, APIQuery{
   298→			Metric:     metric,
   299→			Hostname:   hostname,
   300→			Aggregate:  true,
   301→			Type:       &acceleratorString,
   302→			TypeIds:    accelerators,
   303→			Resolution: resolution,
   304→		})
   305→		scopes = append(scopes, scope)
   306→		return queries, scopes
   307→	}
   308→
   309→	// HWThread -> HWThread
   310→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
   311→		queries = append(queries, APIQuery{
   312→			Metric:     metric,
   313→			Hostname:   hostname,
   314→			Aggregate:  false,
   315→			Type:       &hwthreadString,
   316→			TypeIds:    hwthreadsStr,
   317→			Resolution: resolution,
   318→		})
   319→		scopes = append(scopes, scope)
   320→		return queries, scopes
   321→	}
   322→
   323→	// HWThread -> Core
   324→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
   325→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   326→		for _, core := range cores {
   327→			queries = append(queries, APIQuery{
   328→				Metric:     metric,
   329→				Hostname:   hostname,
   330→				Aggregate:  true,
   331→				Type:       &hwthreadString,
   332→				TypeIds:    intToStringSlice(topology.Core[core]),
   333→				Resolution: resolution,
   334→			})
   335→			scopes = append(scopes, scope)
   336→		}
   337→		return queries, scopes
   338→	}
   339→
   340→	// HWThread -> Socket
   341→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
   342→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   343→		for _, socket := range sockets {
   344→			queries = append(queries, APIQuery{
   345→				Metric:     metric,
   346→				Hostname:   hostname,
   347→				Aggregate:  true,
   348→				Type:       &hwthreadString,
   349→				TypeIds:    intToStringSlice(topology.Socket[socket]),
   350→				Resolution: resolution,
   351→			})
   352→			scopes = append(scopes, scope)
   353→		}
   354→		return queries, scopes
   355→	}
   356→
   357→	// HWThread -> Node
   358→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
   359→		queries = append(queries, APIQuery{
   360→			Metric:     metric,
   361→			Hostname:   hostname,
   362→			Aggregate:  true,
   363→			Type:       &hwthreadString,
   364→			TypeIds:    hwthreadsStr,
   365→			Resolution: resolution,
   366→		})
   367→		scopes = append(scopes, scope)
   368→		return queries, scopes
   369→	}
   370→
   371→	// Core -> Core
   372→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
   373→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   374→		queries = append(queries, APIQuery{
   375→			Metric:     metric,
   376→			Hostname:   hostname,
   377→			Aggregate:  false,
   378→			Type:       &coreString,
   379→			TypeIds:    intToStringSlice(cores),
   380→			Resolution: resolution,
   381→		})
   382→		scopes = append(scopes, scope)
   383→		return queries, scopes
   384→	}
   385→
   386→	// Core -> Socket
   387→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
   388→		sockets, _ := topology.GetSocketsFromCores(hwthreads)
   389→		for _, socket := range sockets {
   390→			queries = append(queries, APIQuery{
   391→				Metric:     metric,
   392→				Hostname:   hostname,
   393→				Aggregate:  true,
   394→				Type:       &coreString,
   395→				TypeIds:    intToStringSlice(topology.Socket[socket]),
   396→				Resolution: resolution,
   397→			})
   398→			scopes = append(scopes, scope)
   399→		}
   400→		return queries, scopes
   401→	}
   402→
   403→	// Core -> Node
   404→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
   405→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   406→		queries = append(queries, APIQuery{
   407→			Metric:     metric,
   408→			Hostname:   hostname,
   409→			Aggregate:  true,
   410→			Type:       &coreString,
   411→			TypeIds:    intToStringSlice(cores),
   412→			Resolution: resolution,
   413→		})
   414→		scopes = append(scopes, scope)
   415→		return queries, scopes
   416→	}
   417→
   418→	// MemoryDomain -> MemoryDomain
   419→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
   420→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   421→		queries = append(queries, APIQuery{
   422→			Metric:     metric,
   423→			Hostname:   hostname,
   424→			Aggregate:  false,
   425→			Type:       &memoryDomainString,
   426→			TypeIds:    intToStringSlice(memDomains),
   427→			Resolution: resolution,
   428→		})
   429→		scopes = append(scopes, scope)
   430→		return queries, scopes
   431→	}
   432→
   433→	// MemoryDomain -> Node
   434→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
   435→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   436→		queries = append(queries, APIQuery{
   437→			Metric:     metric,
   438→			Hostname:   hostname,
   439→			Aggregate:  true,
   440→			Type:       &memoryDomainString,
   441→			TypeIds:    intToStringSlice(memDomains),
   442→			Resolution: resolution,
   443→		})
   444→		scopes = append(scopes, scope)
   445→		return queries, scopes
   446→	}
   447→
   448→	// MemoryDomain -> Socket
   449→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket {
   450→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   451→		socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains)
   452→		if err != nil {
   453→			cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err)
   454→			// Rare Error Case -> Still Continue -> Return Empty Slices
   455→			return queries, scopes
   456→		}
   457→
   458→		// Create a query for each socket
   459→		for _, domains := range socketToDomains {
   460→			queries = append(queries, APIQuery{
   461→				Metric:     metric,
   462→				Hostname:   hostname,
   463→				Aggregate:  true,
   464→				Type:       &memoryDomainString,
   465→				TypeIds:    intToStringSlice(domains),
   466→				Resolution: resolution,
   467→			})
   468→			// Add scope for each query, not just once
   469→			scopes = append(scopes, scope)
   470→		}
   471→		return queries, scopes
   472→	}
   473→
   474→	// Socket -> Socket
   475→	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
   476→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   477→		queries = append(queries, APIQuery{
   478→			Metric:     metric,
   479→			Hostname:   hostname,
   480→			Aggregate:  false,
   481→			Type:       &socketString,
   482→			TypeIds:    intToStringSlice(sockets),
   483→			Resolution: resolution,
   484→		})
   485→		scopes = append(scopes, scope)
   486→		return queries, scopes
   487→	}
   488→
   489→	// Socket -> Node
   490→	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
   491→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   492→		queries = append(queries, APIQuery{
   493→			Metric:     metric,
   494→			Hostname:   hostname,
   495→			Aggregate:  true,
   496→			Type:       &socketString,
   497→			TypeIds:    intToStringSlice(sockets),
   498→			Resolution: resolution,
   499→		})
   500→		scopes = append(scopes, scope)
   501→		return queries, scopes
   502→	}
   503→
   504→	// Node -> Node
   505→	if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
   506→		queries = append(queries, APIQuery{
   507→			Metric:     metric,
   508→			Hostname:   hostname,
   509→			Resolution: resolution,
   510→		})
   511→		scopes = append(scopes, scope)
   512→		return queries, scopes
   513→	}
   514→
   515→	// Unhandled Case -> Error -> Return nils
   516→	return nil, nil
   517→}
   518→
   519→// intToStringSlice converts a slice of integers to a slice of strings.
   520→// Used to convert hardware IDs (core IDs, socket IDs, etc.) to the string format required by the API.
   521→func intToStringSlice(is []int) []string {
   522→	ss := make([]string, len(is))
   523→	for i, x := range is {
   524→		ss[i] = strconv.Itoa(x)
   525→	}
   526→	return ss
   527→}
   528→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→package metricdispatch
     6→
     7→import (
     8→	"bytes"
     9→	"context"
    10→	"encoding/json"
    11→	"fmt"
    12→	"time"
    13→
    14→	"github.com/ClusterCockpit/cc-backend/internal/config"
    15→	ccms "github.com/ClusterCockpit/cc-backend/internal/metricstoreclient"
    16→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    17→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    18→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    19→)
    20→
    21→type MetricDataRepository interface {
    22→	// Return the JobData for the given job, only with the requested metrics.
    23→	LoadData(job *schema.Job,
    24→		metrics []string,
    25→		scopes []schema.MetricScope,
    26→		ctx context.Context,
    27→		resolution int) (schema.JobData, error)
    28→
    29→	// Return a map of metrics to a map of nodes to the metric statistics of the job. node scope only.
    30→	LoadStats(job *schema.Job,
    31→		metrics []string,
    32→		ctx context.Context) (map[string]map[string]schema.MetricStatistics, error)
    33→
    34→	// Return a map of metrics to a map of scopes to the scoped metric statistics of the job.
    35→	LoadScopedStats(job *schema.Job,
    36→		metrics []string,
    37→		scopes []schema.MetricScope,
    38→		ctx context.Context) (schema.ScopedJobStats, error)
    39→
    40→	// Return a map of hosts to a map of metrics at the requested scopes (currently only node) for that node.
    41→	LoadNodeData(cluster string,
    42→		metrics, nodes []string,
    43→		scopes []schema.MetricScope,
    44→		from, to time.Time,
    45→		ctx context.Context) (map[string]map[string][]*schema.JobMetric, error)
    46→
    47→	// Return a map of hosts to a map of metrics to a map of scopes for multiple nodes.
    48→	LoadNodeListData(cluster, subCluster string,
    49→		nodes []string,
    50→		metrics []string,
    51→		scopes []schema.MetricScope,
    52→		resolution int,
    53→		from, to time.Time,
    54→		ctx context.Context) (map[string]schema.JobData, error)
    55→
    56→	// HealthCheck evaluates the monitoring state for a set of nodes against expected metrics.
    57→	HealthCheck(cluster string,
    58→		nodes []string,
    59→		metrics []string) (map[string]metricstore.HealthCheckResult, error)
    60→}
    61→
    62→type CCMetricStoreConfig struct {
    63→	Scope string `json:"scope"`
    64→	URL   string `json:"url"`
    65→	Token string `json:"token"`
    66→}
    67→
    68→var metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{}
    69→
    70→func Init(rawConfig json.RawMessage) error {
    71→	if rawConfig != nil {
    72→		var configs []CCMetricStoreConfig
    73→		config.Validate(configSchema, rawConfig)
    74→		dec := json.NewDecoder(bytes.NewReader(rawConfig))
    75→		dec.DisallowUnknownFields()
    76→		if err := dec.Decode(&configs); err != nil {
    77→			return fmt.Errorf("[METRICDISPATCH]> External Metric Store Config Init: Could not decode config file '%s' Error: %s", rawConfig, err.Error())
    78→		}
    79→
    80→		if len(configs) == 0 {
    81→			return fmt.Errorf("[METRICDISPATCH]> No external metric store configurations found in config file")
    82→		}
    83→
    84→		for _, config := range configs {
    85→			metricDataRepos[config.Scope] = ccms.NewCCMetricStore(config.URL, config.Token)
    86→		}
    87→	}
    88→
    89→	return nil
    90→}
    91→
    92→func GetMetricDataRepo(cluster string, subcluster string) (MetricDataRepository, error) {
    93→	var repo MetricDataRepository
    94→	var ok bool
    95→
    96→	key := cluster + "-" + subcluster
    97→	repo, ok = metricDataRepos[key]
    98→
    99→	if !ok {
   100→		repo, ok = metricDataRepos[cluster]
   101→
   102→		if !ok {
   103→			repo, ok = metricDataRepos["*"]
   104→
   105→			if !ok {
   106→				if metricstore.MetricStoreHandle == nil {
   107→					return nil, fmt.Errorf("[METRICDISPATCH]> no metric data repository configured '%s'", key)
   108→				}
   109→
   110→				repo = metricstore.MetricStoreHandle
   111→				cclog.Debugf("[METRICDISPATCH]> Using internal metric data repository for '%s'", key)
   112→			}
   113→		}
   114→	}
   115→
   116→	return repo, nil
   117→}
   118→
   119→// GetHealthCheckRepo returns the MetricDataRepository for performing health checks on a cluster.
   120→// It uses the same fallback logic as GetMetricDataRepo: cluster → wildcard → internal.
   121→func GetHealthCheckRepo(cluster string) (MetricDataRepository, error) {
   122→	return GetMetricDataRepo(cluster, "")
   123→}
   124→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements high-level query functions for loading job metric data
     7→// with automatic scope transformation and aggregation.
     8→//
     9→// Key Concepts:
    10→//
    11→// Metric Scopes: Metrics are collected at different granularities (native scope):
    12→//   - HWThread: Per hardware thread
    13→//   - Core: Per CPU core
    14→//   - Socket: Per CPU socket
    15→//   - MemoryDomain: Per memory domain (NUMA)
    16→//   - Accelerator: Per GPU/accelerator
    17→//   - Node: Per compute node
    18→//
    19→// Scope Transformation: The buildQueries functions transform between native scope
    20→// and requested scope by:
    21→//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
    22→//   - Rejecting requests for finer granularity than available
    23→//   - Handling special cases (e.g., Accelerator metrics)
    24→//
    25→// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
    26→// based on cluster topology and job resources.
    27→package metricstore
    28→
    29→import (
    30→	"context"
    31→	"fmt"
    32→	"strconv"
    33→	"strings"
    34→	"time"
    35→
    36→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    37→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    38→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    39→)
    40→
    41→type InternalMetricStore struct{}
    42→
    43→var MetricStoreHandle *InternalMetricStore
    44→
    45→// HealthCheck delegates to the internal MemoryStore's HealthCheck.
    46→func (ccms *InternalMetricStore) HealthCheck(cluster string,
    47→	nodes []string, metrics []string,
    48→) (map[string]HealthCheckResult, error) {
    49→	return GetMemoryStore().HealthCheck(cluster, nodes, metrics)
    50→}
    51→
    52→// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
    53→// When set to a non-nil function, LoadData will call this function instead of the default implementation.
    54→var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)
    55→
    56→// LoadData loads metric data for a specific job with automatic scope transformation.
    57→//
    58→// This is the primary function for retrieving job metric data. It handles:
    59→//   - Building queries with scope transformation via buildQueries
    60→//   - Fetching data from the metric store
    61→//   - Organizing results by metric and scope
    62→//   - Converting NaN statistics to 0 for JSON compatibility
    63→//   - Partial error handling (returns data for successful queries even if some fail)
    64→//
    65→// Parameters:
    66→//   - job: Job metadata including cluster, resources, and time range
    67→//   - metrics: List of metric names to load
    68→//   - scopes: Requested metric scopes (will be transformed to match native scopes)
    69→//   - ctx: Context for cancellation (currently unused but reserved for future use)
    70→//   - resolution: Data resolution in seconds (0 for native resolution)
    71→//
    72→// Returns:
    73→//   - JobData: Map of metric → scope → JobMetric with time-series data and statistics
    74→//   - Error: Returns error if query building or fetching fails, or partial error listing failed hosts
    75→//
    76→// Example:
    77→//
    78→//	jobData, err := LoadData(job, []string{"cpu_load", "mem_used"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 60)
    79→func (ccms *InternalMetricStore) LoadData(
    80→	job *schema.Job,
    81→	metrics []string,
    82→	scopes []schema.MetricScope,
    83→	ctx context.Context,
    84→	resolution int,
    85→) (schema.JobData, error) {
    86→	if TestLoadDataCallback != nil {
    87→		return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
    88→	}
    89→
    90→	queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
    91→	if err != nil {
    92→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
    93→		return nil, err
    94→	}
    95→
    96→	// Verify assignment is correct - log any inconsistencies for debugging
    97→	if len(queries) != len(assignedScope) {
    98→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
    99→			len(queries), len(assignedScope))
   100→	}
   101→
   102→	req := APIQueryRequest{
   103→		Cluster:   job.Cluster,
   104→		From:      job.StartTime,
   105→		To:        job.StartTime + int64(job.Duration),
   106→		Queries:   queries,
   107→		WithStats: true,
   108→		WithData:  true,
   109→	}
   110→
   111→	resBody, err := FetchData(req)
   112→	if err != nil {
   113→		cclog.Errorf("Error while fetching data : %s", err.Error())
   114→		return nil, err
   115→	}
   116→
   117→	var errors []string
   118→	jobData := make(schema.JobData)
   119→
   120→	// Add safety check for potential index out of range errors
   121→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   122→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   123→			len(req.Queries), len(resBody.Results), len(assignedScope))
   124→		if len(resBody.Results) > len(req.Queries) {
   125→			resBody.Results = resBody.Results[:len(req.Queries)]
   126→		}
   127→		if len(assignedScope) > len(req.Queries) {
   128→			assignedScope = assignedScope[:len(req.Queries)]
   129→		}
   130→	}
   131→
   132→	for i, row := range resBody.Results {
   133→		// Safety check to prevent index out of range errors
   134→		if i >= len(req.Queries) || i >= len(assignedScope) {
   135→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   136→				i, len(req.Queries), len(assignedScope))
   137→			continue
   138→		}
   139→
   140→		query := req.Queries[i]
   141→		metric := query.Metric
   142→		scope := assignedScope[i]
   143→		mc := archive.GetMetricConfig(job.Cluster, metric)
   144→
   145→		if mc == nil {
   146→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   147→			continue
   148→		}
   149→
   150→		if _, ok := jobData[metric]; !ok {
   151→			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
   152→		}
   153→
   154→		res := mc.Timestep
   155→		if len(row) > 0 {
   156→			res = int(row[0].Resolution)
   157→		}
   158→
   159→		jobMetric, ok := jobData[metric][scope]
   160→		if !ok {
   161→			jobMetric = &schema.JobMetric{
   162→				Unit:     mc.Unit,
   163→				Timestep: res,
   164→				Series:   make([]schema.Series, 0),
   165→			}
   166→			jobData[metric][scope] = jobMetric
   167→		}
   168→
   169→		for ndx, res := range row {
   170→			if res.Error != nil {
   171→				/* Build list for "partial errors", if any */
   172→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   173→				continue
   174→			}
   175→
   176→			id := (*string)(nil)
   177→			if query.Type != nil {
   178→				// Check if ndx is within the bounds of TypeIds slice
   179→				if ndx < len(query.TypeIds) {
   180→					id = new(string)
   181→					*id = query.TypeIds[ndx]
   182→				} else {
   183→					// Log the error but continue processing
   184→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   185→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   186→				}
   187→			}
   188→
   189→			sanitizeStats(&res)
   190→
   191→			jobMetric.Series = append(jobMetric.Series, schema.Series{
   192→				Hostname: query.Hostname,
   193→				ID:       id,
   194→				Statistics: schema.MetricStatistics{
   195→					Avg: float64(res.Avg),
   196→					Min: float64(res.Min),
   197→					Max: float64(res.Max),
   198→				},
   199→				Data: res.Data,
   200→			})


   517→}
   518→
   519→// intToStringSlice converts a slice of integers to a slice of strings.
   520→// Used to convert hardware IDs (core IDs, socket IDs, etc.) to the string format required by the API.
   521→func intToStringSlice(is []int) []string {
   522→	ss := make([]string, len(is))
   523→	for i, x := range is {
   524→		ss[i] = strconv.Itoa(x)
   525→	}
   526→	return ss
   527→}
   528→