     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstoreclient provides a client for querying the cc-metric-store time series database.
     7→//
     8→// The cc-metric-store is a high-performance time series database optimized for HPC metric data.
     9→// This client handles HTTP communication, query construction, scope transformations, and data retrieval
    10→// for job and node metrics across different metric scopes (node, socket, core, hwthread, accelerator).
    11→//
    12→// # Architecture
    13→//
    14→// The package is split into two main components:
    15→//   - Client Operations (cc-metric-store.go): HTTP client, request handling, data loading methods
    16→//   - Query Building (cc-metric-store-queries.go): Query construction and scope transformation logic
    17→//
    18→// # Basic Usage
    19→//
    20→//	store := NewCCMetricStore("http://localhost:8080", "jwt-token")
    21→//
    22→//	// Load job data
    23→//	jobData, err := store.LoadData(job, metrics, scopes, ctx, resolution)
    24→//	if err != nil {
    25→//	    log.Fatal(err)
    26→//	}
    27→//
    28→// # Metric Scopes
    29→//
    30→// The client supports hierarchical metric scopes that map to HPC hardware topology:
    31→//   - MetricScopeAccelerator: GPU/accelerator level metrics
    32→//   - MetricScopeHWThread: Hardware thread (SMT) level metrics
    33→//   - MetricScopeCore: CPU core level metrics
    34→//   - MetricScopeSocket: CPU socket level metrics
    35→//   - MetricScopeMemoryDomain: NUMA domain level metrics
    36→//   - MetricScopeNode: Full node level metrics
    37→//
    38→// The client automatically handles scope transformations, aggregating finer-grained metrics
    39→// to coarser scopes when needed (e.g., aggregating core metrics to socket level).
    40→//
    41→// # Error Handling
    42→//
    43→// The client supports partial errors - if some queries fail, it returns both the successful
    44→// data and an error listing the failed queries. This allows processing partial results
    45→// when some nodes or metrics are temporarily unavailable.
    46→//
    47→// # API Versioning
    48→//
    49→// The client uses cc-metric-store API v2, which includes support for:
    50→//   - Data resampling for bandwidth optimization
    51→//   - Multi-scope queries in a single request
    52→//   - Aggregation across hardware topology levels
    53→package metricstoreclient
    54→
    55→import (
    56→	"bufio"
    57→	"bytes"
    58→	"context"
    59→	"encoding/json"
    60→	"fmt"
    61→	"net/http"
    62→	"strings"
    63→	"time"
    64→
    65→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    66→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    67→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    68→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    69→)
    70→
    71→// CCMetricStore is the HTTP client for communicating with cc-metric-store.
    72→// It manages connection details, authentication, and provides methods for querying metrics.
    73→type CCMetricStore struct {
    74→	client        http.Client                 // HTTP client with 10-second timeout
    75→	jwt           string                      // JWT Bearer token for authentication
    76→	url           string                      // Base URL of cc-metric-store instance
    77→	queryEndpoint string                      // Full URL to query API endpoint
    78→	topologyCache map[string]*schema.Topology // cluster -> topology cache
    79→}
    80→
    81→// APIQueryRequest represents a request to the cc-metric-store query API.
    82→// It supports both explicit queries and "for-all-nodes" bulk queries.
    83→type APIQueryRequest struct {
    84→	Cluster     string     `json:"cluster"`       // Target cluster name
    85→	Queries     []APIQuery `json:"queries"`       // Explicit list of metric queries
    86→	ForAllNodes []string   `json:"for-all-nodes"` // Metrics to query for all nodes
    87→	From        int64      `json:"from"`          // Start time (Unix timestamp)
    88→	To          int64      `json:"to"`            // End time (Unix timestamp)
    89→	WithStats   bool       `json:"with-stats"`    // Include min/avg/max statistics
    90→	WithData    bool       `json:"with-data"`     // Include time series data points
    91→}
    92→
    93→// APIQuery specifies a single metric query with optional scope filtering.
    94→// Type and TypeIds define the hardware scope (core, socket, accelerator, etc.).
    95→type APIQuery struct {
    96→	Type       *string  `json:"type,omitempty"`        // Scope type (e.g., "core", "socket")
    97→	SubType    *string  `json:"subtype,omitempty"`     // Sub-scope type (reserved for future use)
    98→	Metric     string   `json:"metric"`                // Metric name
    99→	Hostname   string   `json:"host"`                  // Target hostname
   100→	Resolution int      `json:"resolution"`            // Data resolution in seconds (0 = native)
   101→	TypeIds    []string `json:"type-ids,omitempty"`    // IDs for the scope type (e.g., core IDs)
   102→	SubTypeIds []string `json:"subtype-ids,omitempty"` // IDs for sub-scope (reserved)
   103→	Aggregate  bool     `json:"aggreg"`                // Aggregate across TypeIds
   104→}
   105→
   106→// APIQueryResponse contains the results from a cc-metric-store query.
   107→// Results align with the Queries slice by index.
   108→type APIQueryResponse struct {
   109→	Queries []APIQuery        `json:"queries,omitempty"` // Echoed queries (for bulk requests)
   110→	Results [][]APIMetricData `json:"results"`           // Result data, indexed by query
   111→}
   112→
   113→// APIMetricData represents time series data and statistics for a single metric series.
   114→// Error is set if this particular series failed to load.
   115→type APIMetricData struct {
   116→	Error      *string        `json:"error"`      // Error message if query failed
   117→	Data       []schema.Float `json:"data"`       // Time series data points
   118→	From       int64          `json:"from"`       // Actual start time of data
   119→	To         int64          `json:"to"`         // Actual end time of data
   120→	Resolution int            `json:"resolution"` // Actual resolution of data in seconds
   121→	Avg        schema.Float   `json:"avg"`        // Average value across time range
   122→	Min        schema.Float   `json:"min"`        // Minimum value in time range
   123→	Max        schema.Float   `json:"max"`        // Maximum value in time range
   124→}
   125→
   126→// NewCCMetricStore creates and initializes a new (external) CCMetricStore client.
   127→// The url parameter should include the protocol and port (e.g., "http://localhost:8080").
   128→// The token parameter is a JWT used for Bearer authentication; pass empty string if auth is disabled.
   129→func NewCCMetricStore(url string, token string) *CCMetricStore {
   130→	return &CCMetricStore{
   131→		url:           url,
   132→		queryEndpoint: fmt.Sprintf("%s/api/query", url),
   133→		jwt:           token,
   134→		client: http.Client{
   135→			Timeout: 10 * time.Second,
   136→		},
   137→		topologyCache: make(map[string]*schema.Topology),
   138→	}
   139→}
   140→
   141→// doRequest executes an HTTP POST request to the cc-metric-store query API.
   142→// It handles JSON encoding/decoding, authentication, and API versioning.
   143→// The request body is automatically closed to prevent resource leaks.
   144→func (ccms *CCMetricStore) doRequest(
   145→	ctx context.Context,
   146→	body *APIQueryRequest,
   147→) (*APIQueryResponse, error) {
   148→	buf := &bytes.Buffer{}
   149→	if err := json.NewEncoder(buf).Encode(body); err != nil {
   150→		cclog.Errorf("Error while encoding request body: %s", err.Error())
   151→		return nil, err
   152→	}
   153→
   154→	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf)
   155→	if err != nil {
   156→		cclog.Errorf("Error while building request body: %s", err.Error())
   157→		return nil, err
   158→	}
   159→	if ccms.jwt != "" {
   160→		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   161→	}
   162→
   163→	// versioning the cc-metric-store query API.
   164→	// v2 = data with resampling
   165→	// v1 = data without resampling
   166→	q := req.URL.Query()
   167→	q.Add("version", "v2")
   168→	req.URL.RawQuery = q.Encode()
   169→
   170→	res, err := ccms.client.Do(req)
   171→	if err != nil {
   172→		cclog.Errorf("Error while performing request: %s", err.Error())
   173→		return nil, err
   174→	}
   175→	defer res.Body.Close()
   176→
   177→	if res.StatusCode != http.StatusOK {
   178→		return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status)
   179→	}
   180→
   181→	var resBody APIQueryResponse
   182→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
   183→		cclog.Errorf("Error while decoding result body: %s", err.Error())
   184→		return nil, err
   185→	}
   186→
   187→	return &resBody, nil
   188→}
   189→
   190→// getTopology returns the topology for a given cluster and subcluster, caching it if not already present
   191→func (ccms *CCMetricStore) getTopology(cluster, subCluster string) (*schema.Topology, error) {
   192→	cacheKey := fmt.Sprintf("%s:%s", cluster, subCluster)
   193→	if topology, ok := ccms.topologyCache[cacheKey]; ok {
   194→		return topology, nil
   195→	}
   196→
   197→	subcluster, err := archive.GetSubCluster(cluster, subCluster)
   198→	if err != nil {
   199→		return nil, err
   200→	}
   201→
   202→	ccms.topologyCache[cacheKey] = &subcluster.Topology
   203→	return &subcluster.Topology, nil
   204→}
   205→
   206→// getTopologyByNode returns the topology for a given cluster and node, caching it if not already present
   207→func (ccms *CCMetricStore) getTopologyByNode(cluster, node string) (*schema.Topology, error) {
   208→	subCluster, err := archive.GetSubClusterByNode(cluster, node)
   209→	if err != nil {
   210→		return nil, err
   211→	}
   212→
   213→	return ccms.getTopology(cluster, subCluster)
   214→}
   215→
   216→// LoadData retrieves time series data and statistics for the specified job and metrics.
   217→// It queries data for the job's time range and resources, handling scope transformations automatically.
   218→//
   219→// Parameters:
   220→//   - job: Job metadata including cluster, time range, and allocated resources
   221→//   - metrics: List of metric names to retrieve
   222→//   - scopes: Requested metric scopes (node, socket, core, etc.)
   223→//   - ctx: Context for cancellation and timeouts
   224→//   - resolution: Data resolution in seconds (0 for native resolution)
   225→//
   226→// Returns JobData organized as: metric -> scope -> series list.
   227→// Supports partial errors: returns available data even if some queries fail.
   228→func (ccms *CCMetricStore) LoadData(
   229→	job *schema.Job,
   230→	metrics []string,
   231→	scopes []schema.MetricScope,
   232→	ctx context.Context,
   233→	resolution int,
   234→) (schema.JobData, error) {
   235→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
   236→	if err != nil {
   237→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   238→		return nil, err
   239→	}
   240→
   241→	// Verify assignment is correct - log any inconsistencies for debugging
   242→	if len(queries) != len(assignedScope) {
   243→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
   244→			len(queries), len(assignedScope))
   245→	}
   246→
   247→	req := APIQueryRequest{
   248→		Cluster:   job.Cluster,
   249→		From:      job.StartTime,
   250→		To:        job.StartTime + int64(job.Duration),
   251→		Queries:   queries,
   252→		WithStats: true,
   253→		WithData:  true,
   254→	}
   255→
   256→	resBody, err := ccms.doRequest(ctx, &req)
   257→	if err != nil {
   258→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   259→		return nil, err
   260→	}
   261→
   262→	var errors []string
   263→	jobData := make(schema.JobData)
   264→
   265→	// Add safety check for potential index out of range errors
   266→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   267→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   268→			len(req.Queries), len(resBody.Results), len(assignedScope))
   269→		if len(resBody.Results) > len(req.Queries) {
   270→			resBody.Results = resBody.Results[:len(req.Queries)]
   271→		}
   272→		if len(assignedScope) > len(req.Queries) {
   273→			assignedScope = assignedScope[:len(req.Queries)]
   274→		}
   275→	}
   276→
   277→	for i, row := range resBody.Results {
   278→		// Safety check to prevent index out of range errors
   279→		if i >= len(req.Queries) || i >= len(assignedScope) {
   280→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   281→				i, len(req.Queries), len(assignedScope))
   282→			continue
   283→		}
   284→
   285→		query := req.Queries[i]
   286→		metric := query.Metric
   287→		scope := assignedScope[i]
   288→		mc := archive.GetMetricConfig(job.Cluster, metric)
   289→
   290→		if mc == nil {
   291→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   292→			continue
   293→		}
   294→
   295→		if _, ok := jobData[metric]; !ok {
   296→			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
   297→		}
   298→
   299→		res := mc.Timestep
   300→		if len(row) > 0 {
   301→			res = row[0].Resolution
   302→		}
   303→
   304→		jobMetric, ok := jobData[metric][scope]
   305→		if !ok {
   306→			jobMetric = &schema.JobMetric{
   307→				Unit:     mc.Unit,
   308→				Timestep: res,
   309→				Series:   make([]schema.Series, 0),
   310→			}
   311→			jobData[metric][scope] = jobMetric
   312→		}
   313→
   314→		for ndx, res := range row {
   315→			if res.Error != nil {
   316→				/* Build list for "partial errors", if any */
   317→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   318→				continue
   319→			}
   320→
   321→			id := (*string)(nil)
   322→			if query.Type != nil {
   323→				// Check if ndx is within the bounds of TypeIds slice
   324→				if ndx < len(query.TypeIds) {
   325→					id = new(string)
   326→					*id = query.TypeIds[ndx]
   327→				} else {
   328→					// Log the error but continue processing
   329→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   330→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   331→				}
   332→			}
   333→
   334→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   335→
   336→			jobMetric.Series = append(jobMetric.Series, schema.Series{
   337→				Hostname: query.Hostname,
   338→				ID:       id,
   339→				Statistics: schema.MetricStatistics{
   340→					Avg: float64(res.Avg),
   341→					Min: float64(res.Min),
   342→					Max: float64(res.Max),
   343→				},
   344→				Data: res.Data,
   345→			})
   346→		}
   347→
   348→		// So that one can later check len(jobData):
   349→		if len(jobMetric.Series) == 0 {
   350→			delete(jobData[metric], scope)
   351→			if len(jobData[metric]) == 0 {
   352→				delete(jobData, metric)
   353→			}
   354→		}
   355→	}
   356→
   357→	if len(errors) != 0 {
   358→		/* Returns list for "partial errors" */
   359→		return jobData, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   360→	}
   361→	return jobData, nil
   362→}
   363→
   364→// LoadStats retrieves min/avg/max statistics for job metrics at node scope.
   365→// This is faster than LoadData when only statistical summaries are needed (no time series data).
   366→//
   367→// Returns statistics organized as: metric -> hostname -> statistics.
   368→func (ccms *CCMetricStore) LoadStats(
   369→	job *schema.Job,
   370→	metrics []string,
   371→	ctx context.Context,
   372→) (map[string]map[string]schema.MetricStatistics, error) {
   373→	queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization?
   374→	if err != nil {
   375→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
   376→		return nil, err
   377→	}
   378→
   379→	req := APIQueryRequest{
   380→		Cluster:   job.Cluster,
   381→		From:      job.StartTime,
   382→		To:        job.StartTime + int64(job.Duration),
   383→		Queries:   queries,
   384→		WithStats: true,
   385→		WithData:  false,
   386→	}
   387→
   388→	resBody, err := ccms.doRequest(ctx, &req)
   389→	if err != nil {
   390→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   391→		return nil, err
   392→	}
   393→
   394→	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
   395→	for i, res := range resBody.Results {
   396→		if len(res) == 0 {
   397→			// No Data Found For Metric, Logged in FetchData to Warn
   398→			continue
   399→		}
   400→		query := req.Queries[i]
   401→		metric := query.Metric
   402→		data := res[0]
   403→		if data.Error != nil {
   404→			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
   405→			continue
   406→		}
   407→
   408→		metricdata, ok := stats[metric]
   409→		if !ok {
   410→			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
   411→			stats[metric] = metricdata
   412→		}
   413→
   414→		if hasNaNStats(data.Avg, data.Min, data.Max) {
   415→			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
   416→			continue
   417→		}
   418→
   419→		metricdata[query.Hostname] = schema.MetricStatistics{
   420→			Avg: float64(data.Avg),
   421→			Min: float64(data.Min),
   422→			Max: float64(data.Max),
   423→		}
   424→	}
   425→
   426→	return stats, nil
   427→}
   428→
   429→// LoadScopedStats retrieves statistics for job metrics across multiple scopes.
   430→// Used for the Job-View Statistics Table to display per-scope breakdowns.
   431→//
   432→// Returns statistics organized as: metric -> scope -> list of scoped statistics.
   433→// Each scoped statistic includes hostname, hardware ID (if applicable), and min/avg/max values.
   434→func (ccms *CCMetricStore) LoadScopedStats(
   435→	job *schema.Job,
   436→	metrics []string,
   437→	scopes []schema.MetricScope,
   438→	ctx context.Context,
   439→) (schema.ScopedJobStats, error) {
   440→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
   441→	if err != nil {
   442→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   443→		return nil, err
   444→	}
   445→
   446→	req := APIQueryRequest{
   447→		Cluster:   job.Cluster,
   448→		From:      job.StartTime,
   449→		To:        job.StartTime + int64(job.Duration),
   450→		Queries:   queries,
   451→		WithStats: true,
   452→		WithData:  false,
   453→	}
   454→
   455→	resBody, err := ccms.doRequest(ctx, &req)
   456→	if err != nil {
   457→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   458→		return nil, err
   459→	}
   460→
   461→	var errors []string
   462→	scopedJobStats := make(schema.ScopedJobStats)
   463→
   464→	for i, row := range resBody.Results {
   465→		query := req.Queries[i]
   466→		metric := query.Metric
   467→		scope := assignedScope[i]
   468→
   469→		if _, ok := scopedJobStats[metric]; !ok {
   470→			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
   471→		}
   472→
   473→		if _, ok := scopedJobStats[metric][scope]; !ok {
   474→			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
   475→		}
   476→
   477→		for ndx, res := range row {
   478→			if res.Error != nil {
   479→				/* Build list for "partial errors", if any */
   480→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   481→				continue
   482→			}
   483→
   484→			id := (*string)(nil)
   485→			if query.Type != nil {
   486→				// Check if ndx is within the bounds of TypeIds slice
   487→				if ndx < len(query.TypeIds) {
   488→					id = new(string)
   489→					*id = query.TypeIds[ndx]
   490→				} else {
   491→					// Log the error but continue processing
   492→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   493→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   494→				}
   495→			}
   496→
   497→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   498→
   499→			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
   500→				Hostname: query.Hostname,
   501→				ID:       id,
   502→				Data: &schema.MetricStatistics{
   503→					Avg: float64(res.Avg),
   504→					Min: float64(res.Min),
   505→					Max: float64(res.Max),
   506→				},
   507→			})
   508→		}
   509→
   510→		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
   511→		if len(scopedJobStats[metric][scope]) == 0 {
   512→			delete(scopedJobStats[metric], scope)
   513→			if len(scopedJobStats[metric]) == 0 {
   514→				delete(scopedJobStats, metric)
   515→			}
   516→		}
   517→	}
   518→
   519→	if len(errors) != 0 {
   520→		/* Returns list for "partial errors" */
   521→		return scopedJobStats, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   522→	}
   523→	return scopedJobStats, nil
   524→}
   525→
   526→// LoadNodeData retrieves current metric data for specified nodes in a cluster.
   527→// Used for the Systems-View Node-Overview to display real-time node status.
   528→//
   529→// If nodes is nil, queries all metrics for all nodes in the cluster (bulk query).
   530→// Returns data organized as: hostname -> metric -> list of JobMetric (with time series and stats).
   531→func (ccms *CCMetricStore) LoadNodeData(
   532→	cluster string,
   533→	metrics, nodes []string,
   534→	scopes []schema.MetricScope,
   535→	from, to time.Time,
   536→	ctx context.Context,
   537→) (map[string]map[string][]*schema.JobMetric, error) {
   538→	req := APIQueryRequest{
   539→		Cluster:   cluster,
   540→		From:      from.Unix(),
   541→		To:        to.Unix(),
   542→		WithStats: true,
   543→		WithData:  true,
   544→	}
   545→
   546→	if nodes == nil {
   547→		req.ForAllNodes = append(req.ForAllNodes, metrics...)
   548→	} else {
   549→		for _, node := range nodes {
   550→			for _, metric := range metrics {
   551→				req.Queries = append(req.Queries, APIQuery{
   552→					Hostname:   node,
   553→					Metric:     metric,
   554→					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
   555→				})
   556→			}
   557→		}
   558→	}
   559→
   560→	resBody, err := ccms.doRequest(ctx, &req)
   561→	if err != nil {
   562→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   563→		return nil, err
   564→	}
   565→
   566→	var errors []string
   567→	data := make(map[string]map[string][]*schema.JobMetric)
   568→	for i, res := range resBody.Results {
   569→		if len(res) == 0 {
   570→			// No Data Found For Metric, Logged in FetchData to Warn
   571→			continue
   572→		}
   573→
   574→		var query APIQuery
   575→		if resBody.Queries != nil {
   576→			query = resBody.Queries[i]
   577→		} else {
   578→			query = req.Queries[i]
   579→		}
   580→
   581→		metric := query.Metric
   582→		qdata := res[0]
   583→		if qdata.Error != nil {
   584→			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
   585→		}
   586→
   587→		sanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)
   588→
   589→		hostdata, ok := data[query.Hostname]
   590→		if !ok {
   591→			hostdata = make(map[string][]*schema.JobMetric)
   592→			data[query.Hostname] = hostdata
   593→		}
   594→
   595→		mc := archive.GetMetricConfig(cluster, metric)
   596→		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
   597→			Unit:     mc.Unit,
   598→			Timestep: mc.Timestep,
   599→			Series: []schema.Series{
   600→				{
   601→					Hostname: query.Hostname,
   602→					Data:     qdata.Data,
   603→					Statistics: schema.MetricStatistics{
   604→						Avg: float64(qdata.Avg),
   605→						Min: float64(qdata.Min),
   606→						Max: float64(qdata.Max),
   607→					},
   608→				},
   609→			},
   610→		})
   611→	}
   612→
   613→	if len(errors) != 0 {
   614→		/* Returns list of "partial errors" */
   615→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   616→	}
   617→
   618→	return data, nil
   619→}
   620→
   621→// LoadNodeListData retrieves paginated node metrics for the Systems-View Node-List.
   622→//
   623→// Supports filtering by subcluster and node name pattern. The nodeFilter performs
   624→// substring matching on hostnames.
   625→//
   626→// Returns:
   627→//   - Node data organized as: hostname -> JobData (metric -> scope -> series)
   628→//   - Total node count (before pagination)
   629→//   - HasNextPage flag indicating if more pages are available
   630→//   - Error (may be partial error with some data returned)
   631→func (ccms *CCMetricStore) LoadNodeListData(
   632→	cluster, subCluster string,
   633→	nodes []string,
   634→	metrics []string,
   635→	scopes []schema.MetricScope,
   636→	resolution int,
   637→	from, to time.Time,
   638→	ctx context.Context,
   639→) (map[string]schema.JobData, error) {
   640→	queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
   641→	if err != nil {
   642→		cclog.Errorf("Error while building node queries for Cluster %s, SubCluster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
   643→		return nil, err
   644→	}
   645→
   646→	// Verify assignment is correct - log any inconsistencies for debugging
   647→	if len(queries) != len(assignedScope) {
   648→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
   649→			len(queries), len(assignedScope))
   650→	}
   651→
   652→	req := APIQueryRequest{
   653→		Cluster:   cluster,
   654→		Queries:   queries,
   655→		From:      from.Unix(),
   656→		To:        to.Unix(),
   657→		WithStats: true,
   658→		WithData:  true,
   659→	}
   660→
   661→	resBody, err := ccms.doRequest(ctx, &req)
   662→	if err != nil {
   663→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   664→		return nil, err
   665→	}
   666→
   667→	var errors []string
   668→	data := make(map[string]schema.JobData)
   669→
   670→	// Add safety check for index out of range issues
   671→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   672→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   673→			len(req.Queries), len(resBody.Results), len(assignedScope))
   674→		if len(resBody.Results) > len(req.Queries) {
   675→			resBody.Results = resBody.Results[:len(req.Queries)]
   676→		}
   677→		if len(assignedScope) > len(req.Queries) {
   678→			assignedScope = assignedScope[:len(req.Queries)]
   679→		}
   680→	}
   681→
   682→	for i, row := range resBody.Results {
   683→		// Safety check to prevent index out of range errors
   684→		if i >= len(req.Queries) || i >= len(assignedScope) {
   685→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   686→				i, len(req.Queries), len(assignedScope))
   687→			continue
   688→		}
   689→
   690→		var query APIQuery
   691→		if resBody.Queries != nil {
   692→			if i < len(resBody.Queries) {
   693→				query = resBody.Queries[i]
   694→			} else {
   695→				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
   696→					i, len(resBody.Queries))
   697→				continue
   698→			}
   699→		} else {
   700→			query = req.Queries[i]
   701→		}
   702→
   703→		metric := query.Metric
   704→		scope := assignedScope[i]
   705→		mc := archive.GetMetricConfig(cluster, metric)
   706→		if mc == nil {
   707→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   708→			continue
   709→		}
   710→
   711→		res := mc.Timestep
   712→		if len(row) > 0 {
   713→			res = row[0].Resolution
   714→		}
   715→
   716→		// Init Nested Map Data Structures If Not Found
   717→		hostData, ok := data[query.Hostname]
   718→		if !ok {
   719→			hostData = make(schema.JobData)
   720→			data[query.Hostname] = hostData
   721→		}
   722→
   723→		metricData, ok := hostData[metric]
   724→		if !ok {
   725→			metricData = make(map[schema.MetricScope]*schema.JobMetric)
   726→			data[query.Hostname][metric] = metricData
   727→		}
   728→
   729→		scopeData, ok := metricData[scope]
   730→		if !ok {
   731→			scopeData = &schema.JobMetric{
   732→				Unit:     mc.Unit,
   733→				Timestep: res,
   734→				Series:   make([]schema.Series, 0),
   735→			}
   736→			data[query.Hostname][metric][scope] = scopeData
   737→		}
   738→
   739→		for ndx, res := range row {
   740→			if res.Error != nil {
   741→				/* Build list for "partial errors", if any */
   742→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   743→				continue
   744→			}
   745→
   746→			id := (*string)(nil)
   747→			if query.Type != nil {
   748→				// Check if ndx is within the bounds of TypeIds slice
   749→				if ndx < len(query.TypeIds) {
   750→					id = new(string)
   751→					*id = query.TypeIds[ndx]
   752→				} else {
   753→					// Log the error but continue processing
   754→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   755→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   756→				}
   757→			}
   758→
   759→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   760→
   761→			scopeData.Series = append(scopeData.Series, schema.Series{
   762→				Hostname: query.Hostname,
   763→				ID:       id,
   764→				Statistics: schema.MetricStatistics{
   765→					Avg: float64(res.Avg),
   766→					Min: float64(res.Min),
   767→					Max: float64(res.Max),
   768→				},
   769→				Data: res.Data,
   770→			})
   771→		}
   772→	}
   773→
   774→	if len(errors) != 0 {
   775→		/* Returns list of "partial errors" */
   776→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   777→	}
   778→
   779→	return data, nil
   780→}
   781→
   782→// HealthCheck queries the external cc-metric-store's health check endpoint.
   783→// It sends a HealthCheckReq as the request body to /api/healthcheck and
   784→// returns the per-node health check results.
   785→func (ccms *CCMetricStore) HealthCheck(cluster string,
   786→	nodes []string, metrics []string,
   787→) (map[string]metricstore.HealthCheckResult, error) {
   788→	req := metricstore.HealthCheckReq{
   789→		Cluster:     cluster,
   790→		Nodes:       nodes,
   791→		MetricNames: metrics,
   792→	}
   793→
   794→	buf := &bytes.Buffer{}
   795→	if err := json.NewEncoder(buf).Encode(req); err != nil {
   796→		cclog.Errorf("Error while encoding health check request body: %s", err.Error())
   797→		return nil, err
   798→	}
   799→
   800→	endpoint := fmt.Sprintf("%s/api/healthcheck", ccms.url)
   801→	httpReq, err := http.NewRequest(http.MethodGet, endpoint, buf)
   802→	if err != nil {
   803→		cclog.Errorf("Error while building health check request: %s", err.Error())
   804→		return nil, err
   805→	}
   806→	if ccms.jwt != "" {
   807→		httpReq.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   808→	}
   809→
   810→	res, err := ccms.client.Do(httpReq)
   811→	if err != nil {
   812→		cclog.Errorf("Error while performing health check request: %s", err.Error())
   813→		return nil, err
   814→	}
   815→	defer res.Body.Close()
   816→
   817→	if res.StatusCode != http.StatusOK {
   818→		return nil, fmt.Errorf("'%s': HTTP Status: %s", endpoint, res.Status)
   819→	}
   820→
   821→	var results map[string]metricstore.HealthCheckResult
   822→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&results); err != nil {
   823→		cclog.Errorf("Error while decoding health check response: %s", err.Error())
   824→		return nil, err
   825→	}
   826→
   827→	return results, nil
   828→}
   829→
   830→// sanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
   831→// Regular float64 values cannot be JSONed when NaN.
   832→func sanitizeStats(avg, min, max *schema.Float) {
   833→	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
   834→		*avg = schema.Float(0)
   835→		*min = schema.Float(0)
   836→		*max = schema.Float(0)
   837→	}
   838→}
   839→
   840→// hasNaNStats returns true if any of the statistics contain NaN values.
   841→func hasNaNStats(avg, min, max schema.Float) bool {
   842→	return avg.IsNaN() || min.IsNaN() || max.IsNaN()
   843→}
   844→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstoreclient - Query Building
     7→//
     8→// This file contains the query construction and scope transformation logic for cc-metric-store queries.
     9→// It handles the complex mapping between requested metric scopes and native hardware topology,
    10→// automatically aggregating or filtering metrics as needed.
    11→//
    12→// # Scope Transformations
    13→//
    14→// The buildScopeQueries function implements the core scope transformation algorithm.
    15→// It handles 25+ different transformation cases, mapping between:
    16→//   - Accelerator (GPU) scope
    17→//   - HWThread (hardware thread/SMT) scope
    18→//   - Core (CPU core) scope
    19→//   - Socket (CPU package) scope
    20→//   - MemoryDomain (NUMA domain) scope
    21→//   - Node (full system) scope
    22→//
    23→// Transformations follow these rules:
    24→//   - Same scope: Return data as-is (e.g., Core → Core)
    25→//   - Coarser scope: Aggregate data (e.g., Core → Socket with Aggregate=true)
    26→//   - Finer scope: Error - cannot increase granularity
    27→//
    28→// # Query Building
    29→//
    30→// buildQueries and buildNodeQueries are the main entry points, handling job-specific
    31→// and node-specific query construction respectively. They:
    32→//   - Validate metric configurations
    33→//   - Handle subcluster-specific metric filtering
    34→//   - Detect and skip duplicate scope requests
    35→//   - Call buildScopeQueries for each metric/scope/host combination
    36→package metricstoreclient
    37→
    38→import (
    39→	"fmt"
    40→	"strconv"
    41→
    42→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    43→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    44→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    45→)
    46→
    47→// Scope string constants used in API queries.
    48→// Pre-converted to avoid repeated allocations during query building.
    49→var (
    50→	hwthreadString     = string(schema.MetricScopeHWThread)
    51→	coreString         = string(schema.MetricScopeCore)
    52→	memoryDomainString = string(schema.MetricScopeMemoryDomain)
    53→	socketString       = string(schema.MetricScopeSocket)
    54→	acceleratorString  = string(schema.MetricScopeAccelerator)
    55→)
    56→
    57→// buildQueries constructs API queries for job-specific metric data.
    58→// It iterates through metrics, scopes, and job resources to build the complete query set.
    59→//
    60→// The function handles:
    61→//   - Metric configuration validation and subcluster filtering
    62→//   - Scope deduplication to avoid redundant queries
    63→//   - Hardware thread list resolution (job-allocated vs full node)
    64→//   - Delegation to buildScopeQueries for scope transformations
    65→//
    66→// Returns queries and their corresponding assigned scopes (which may differ from requested scopes).
    67→func (ccms *CCMetricStore) buildQueries(
    68→	job *schema.Job,
    69→	metrics []string,
    70→	scopes []schema.MetricScope,
    71→	resolution int,
    72→) ([]APIQuery, []schema.MetricScope, error) {
    73→	// Initialize both slices together
    74→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
    75→	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(job.Resources))
    76→
    77→	topology, err := ccms.getTopology(job.Cluster, job.SubCluster)
    78→	if err != nil {
    79→		cclog.Errorf("could not load cluster %s subCluster %s topology: %s", job.Cluster, job.SubCluster, err.Error())
    80→		return nil, nil, err
    81→	}
    82→
    83→	for _, metric := range metrics {
    84→		remoteName := metric
    85→		mc := archive.GetMetricConfig(job.Cluster, metric)
    86→		if mc == nil {
    87→			cclog.Warnf("metric '%s' is not specified for cluster '%s' - skipping", metric, job.Cluster)
    88→			continue
    89→		}
    90→
    91→		// Skip if metric is removed for subcluster
    92→		if len(mc.SubClusters) != 0 {
    93→			isRemoved := false
    94→			for _, scConfig := range mc.SubClusters {
    95→				if scConfig.Name == job.SubCluster && scConfig.Remove {
    96→					isRemoved = true
    97→					break
    98→				}
    99→			}
   100→			if isRemoved {
   101→				continue
   102→			}
   103→		}
   104→
   105→		// Avoid duplicates...
   106→		handledScopes := make([]schema.MetricScope, 0, 3)
   107→
   108→	scopesLoop:
   109→		for _, requestedScope := range scopes {
   110→			nativeScope := mc.Scope
   111→			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
   112→				continue
   113→			}
   114→
   115→			scope := nativeScope.Max(requestedScope)
   116→			for _, s := range handledScopes {
   117→				if scope == s {
   118→					continue scopesLoop
   119→				}
   120→			}
   121→			handledScopes = append(handledScopes, scope)
   122→
   123→			for _, host := range job.Resources {
   124→				hwthreads := host.HWThreads
   125→				if hwthreads == nil {
   126→					hwthreads = topology.Node
   127→				}
   128→
   129→				// Note: Expected exceptions will return as empty slices -> Continue
   130→				hostQueries, hostScopes := buildScopeQueries(
   131→					nativeScope, requestedScope,
   132→					remoteName, host.Hostname,
   133→					topology, hwthreads, host.Accelerators,
   134→					resolution,
   135→				)
   136→
   137→				// Note: Unexpected errors, such as unhandled cases, will return as nils -> Error
   138→				if hostQueries == nil && hostScopes == nil {
   139→					return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   140→				}
   141→
   142→				queries = append(queries, hostQueries...)
   143→				assignedScope = append(assignedScope, hostScopes...)
   144→			}
   145→		}
   146→	}
   147→
   148→	return queries, assignedScope, nil
   149→}
   150→
   151→// buildNodeQueries constructs API queries for node-specific metric data (Systems View).
   152→// Similar to buildQueries but uses full node topology instead of job-allocated resources.
   153→//
   154→// The function handles:
   155→//   - SubCluster topology resolution (either pre-loaded or per-node lookup)
   156→//   - Full node hardware thread lists (not job-specific subsets)
   157→//   - All accelerators on each node
   158→//   - Metric configuration validation with subcluster filtering
   159→//
   160→// Returns queries and their corresponding assigned scopes.
   161→func (ccms *CCMetricStore) buildNodeQueries(
   162→	cluster string,
   163→	subCluster string,
   164→	nodes []string,
   165→	metrics []string,
   166→	scopes []schema.MetricScope,
   167→	resolution int,
   168→) ([]APIQuery, []schema.MetricScope, error) {
   169→	// Initialize both slices together
   170→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
   171→	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(nodes))
   172→
   173→	for _, metric := range metrics {
   174→		remoteName := metric
   175→		mc := archive.GetMetricConfig(cluster, metric)
   176→		if mc == nil {
   177→			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
   178→			continue
   179→		}
   180→
   181→		// Skip if metric is removed for subcluster
   182→		if mc.SubClusters != nil {
   183→			isRemoved := false
   184→			for _, scConfig := range mc.SubClusters {
   185→				if scConfig.Name == subCluster && scConfig.Remove {
   186→					isRemoved = true
   187→					break
   188→				}
   189→			}
   190→			if isRemoved {
   191→				continue
   192→			}
   193→		}
   194→
   195→		// Avoid duplicates...
   196→		handledScopes := make([]schema.MetricScope, 0, 3)
   197→
   198→	scopesLoop:
   199→		for _, requestedScope := range scopes {
   200→			nativeScope := mc.Scope
   201→
   202→			scope := nativeScope.Max(requestedScope)
   203→			for _, s := range handledScopes {
   204→				if scope == s {
   205→					continue scopesLoop
   206→				}
   207→			}
   208→			handledScopes = append(handledScopes, scope)
   209→
   210→			for _, hostname := range nodes {
   211→				var topology *schema.Topology
   212→				var err error
   213→
   214→				// If no subCluster given, get it by node
   215→				if subCluster == "" {
   216→					topology, err = ccms.getTopologyByNode(cluster, hostname)
   217→				} else {
   218→					topology, err = ccms.getTopology(cluster, subCluster)
   219→				}
   220→
   221→				if err != nil {
   222→					return nil, nil, err
   223→				}
   224→
   225→				// Always full node hwthread id list, no partial queries expected -> Use "topology.Node" directly where applicable
   226→				// Always full accelerator id list, no partial queries expected -> Use "acceleratorIds" directly where applicable
   227→				acceleratorIds := topology.GetAcceleratorIDs()
   228→
   229→				// Moved check here if metric matches hardware specs
   230→				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
   231→					continue scopesLoop
   232→				}
   233→
   234→				nodeQueries, nodeScopes := buildScopeQueries(
   235→					nativeScope, requestedScope,
   236→					remoteName, hostname,
   237→					topology, topology.Node, acceleratorIds,
   238→					resolution,
   239→				)
   240→
   241→				if len(nodeQueries) == 0 && len(nodeScopes) == 0 {
   242→					return nil, nil, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > TODO: unhandled case: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   243→				}
   244→
   245→				queries = append(queries, nodeQueries...)
   246→				assignedScope = append(assignedScope, nodeScopes...)
   247→			}
   248→		}
   249→	}
   250→
   251→	return queries, assignedScope, nil
   252→}
   253→
   254→// buildScopeQueries generates API queries for a given scope transformation.
   255→// It returns a slice of queries and corresponding assigned scopes.
   256→// Some transformations (e.g., HWThread -> Core/Socket) may generate multiple queries.
   257→func buildScopeQueries(
   258→	nativeScope, requestedScope schema.MetricScope,
   259→	metric, hostname string,
   260→	topology *schema.Topology,
   261→	hwthreads []int,
   262→	accelerators []string,
   263→	resolution int,
   264→) ([]APIQuery, []schema.MetricScope) {
   265→	scope := nativeScope.Max(requestedScope)
   266→	queries := []APIQuery{}
   267→	scopes := []schema.MetricScope{}
   268→
   269→	hwthreadsStr := intToStringSlice(hwthreads)
   270→
   271→	// Accelerator -> Accelerator (Use "accelerator" scope if requested scope is lower than node)
   272→	if nativeScope == schema.MetricScopeAccelerator && scope.LT(schema.MetricScopeNode) {
   273→		if scope != schema.MetricScopeAccelerator {
   274→			// Expected Exception -> Continue -> Return Empty Slices
   275→			return queries, scopes
   276→		}
   277→
   278→		queries = append(queries, APIQuery{
   279→			Metric:     metric,
   280→			Hostname:   hostname,
   281→			Aggregate:  false,
   282→			Type:       &acceleratorString,
   283→			TypeIds:    accelerators,
   284→			Resolution: resolution,
   285→		})
   286→		scopes = append(scopes, schema.MetricScopeAccelerator)
   287→		return queries, scopes
   288→	}
   289→
   290→	// Accelerator -> Node
   291→	if nativeScope == schema.MetricScopeAccelerator && scope == schema.MetricScopeNode {
   292→		if len(accelerators) == 0 {
   293→			// Expected Exception -> Continue -> Return Empty Slices
   294→			return queries, scopes
   295→		}
   296→
   297→		queries = append(queries, APIQuery{
   298→			Metric:     metric,
   299→			Hostname:   hostname,
   300→			Aggregate:  true,
   301→			Type:       &acceleratorString,
   302→			TypeIds:    accelerators,
   303→			Resolution: resolution,
   304→		})
   305→		scopes = append(scopes, scope)
   306→		return queries, scopes
   307→	}
   308→
   309→	// HWThread -> HWThread
   310→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeHWThread {
   311→		queries = append(queries, APIQuery{
   312→			Metric:     metric,
   313→			Hostname:   hostname,
   314→			Aggregate:  false,
   315→			Type:       &hwthreadString,
   316→			TypeIds:    hwthreadsStr,
   317→			Resolution: resolution,
   318→		})
   319→		scopes = append(scopes, scope)
   320→		return queries, scopes
   321→	}
   322→
   323→	// HWThread -> Core
   324→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeCore {
   325→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   326→		for _, core := range cores {
   327→			queries = append(queries, APIQuery{
   328→				Metric:     metric,
   329→				Hostname:   hostname,
   330→				Aggregate:  true,
   331→				Type:       &hwthreadString,
   332→				TypeIds:    intToStringSlice(topology.Core[core]),
   333→				Resolution: resolution,
   334→			})
   335→			scopes = append(scopes, scope)
   336→		}
   337→		return queries, scopes
   338→	}
   339→
   340→	// HWThread -> Socket
   341→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeSocket {
   342→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   343→		for _, socket := range sockets {
   344→			queries = append(queries, APIQuery{
   345→				Metric:     metric,
   346→				Hostname:   hostname,
   347→				Aggregate:  true,
   348→				Type:       &hwthreadString,
   349→				TypeIds:    intToStringSlice(topology.Socket[socket]),
   350→				Resolution: resolution,
   351→			})
   352→			scopes = append(scopes, scope)
   353→		}
   354→		return queries, scopes
   355→	}
   356→
   357→	// HWThread -> Node
   358→	if nativeScope == schema.MetricScopeHWThread && scope == schema.MetricScopeNode {
   359→		queries = append(queries, APIQuery{
   360→			Metric:     metric,
   361→			Hostname:   hostname,
   362→			Aggregate:  true,
   363→			Type:       &hwthreadString,
   364→			TypeIds:    hwthreadsStr,
   365→			Resolution: resolution,
   366→		})
   367→		scopes = append(scopes, scope)
   368→		return queries, scopes
   369→	}
   370→
   371→	// Core -> Core
   372→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeCore {
   373→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   374→		queries = append(queries, APIQuery{
   375→			Metric:     metric,
   376→			Hostname:   hostname,
   377→			Aggregate:  false,
   378→			Type:       &coreString,
   379→			TypeIds:    intToStringSlice(cores),
   380→			Resolution: resolution,
   381→		})
   382→		scopes = append(scopes, scope)
   383→		return queries, scopes
   384→	}
   385→
   386→	// Core -> Socket
   387→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeSocket {
   388→		sockets, _ := topology.GetSocketsFromCores(hwthreads)
   389→		for _, socket := range sockets {
   390→			queries = append(queries, APIQuery{
   391→				Metric:     metric,
   392→				Hostname:   hostname,
   393→				Aggregate:  true,
   394→				Type:       &coreString,
   395→				TypeIds:    intToStringSlice(topology.Socket[socket]),
   396→				Resolution: resolution,
   397→			})
   398→			scopes = append(scopes, scope)
   399→		}
   400→		return queries, scopes
   401→	}
   402→
   403→	// Core -> Node
   404→	if nativeScope == schema.MetricScopeCore && scope == schema.MetricScopeNode {
   405→		cores, _ := topology.GetCoresFromHWThreads(hwthreads)
   406→		queries = append(queries, APIQuery{
   407→			Metric:     metric,
   408→			Hostname:   hostname,
   409→			Aggregate:  true,
   410→			Type:       &coreString,
   411→			TypeIds:    intToStringSlice(cores),
   412→			Resolution: resolution,
   413→		})
   414→		scopes = append(scopes, scope)
   415→		return queries, scopes
   416→	}
   417→
   418→	// MemoryDomain -> MemoryDomain
   419→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeMemoryDomain {
   420→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   421→		queries = append(queries, APIQuery{
   422→			Metric:     metric,
   423→			Hostname:   hostname,
   424→			Aggregate:  false,
   425→			Type:       &memoryDomainString,
   426→			TypeIds:    intToStringSlice(memDomains),
   427→			Resolution: resolution,
   428→		})
   429→		scopes = append(scopes, scope)
   430→		return queries, scopes
   431→	}
   432→
   433→	// MemoryDomain -> Node
   434→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeNode {
   435→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   436→		queries = append(queries, APIQuery{
   437→			Metric:     metric,
   438→			Hostname:   hostname,
   439→			Aggregate:  true,
   440→			Type:       &memoryDomainString,
   441→			TypeIds:    intToStringSlice(memDomains),
   442→			Resolution: resolution,
   443→		})
   444→		scopes = append(scopes, scope)
   445→		return queries, scopes
   446→	}
   447→
   448→	// MemoryDomain -> Socket
   449→	if nativeScope == schema.MetricScopeMemoryDomain && scope == schema.MetricScopeSocket {
   450→		memDomains, _ := topology.GetMemoryDomainsFromHWThreads(hwthreads)
   451→		socketToDomains, err := topology.GetMemoryDomainsBySocket(memDomains)
   452→		if err != nil {
   453→			cclog.Errorf("Error mapping memory domains to sockets, return unchanged: %v", err)
   454→			// Rare Error Case -> Still Continue -> Return Empty Slices
   455→			return queries, scopes
   456→		}
   457→
   458→		// Create a query for each socket
   459→		for _, domains := range socketToDomains {
   460→			queries = append(queries, APIQuery{
   461→				Metric:     metric,
   462→				Hostname:   hostname,
   463→				Aggregate:  true,
   464→				Type:       &memoryDomainString,
   465→				TypeIds:    intToStringSlice(domains),
   466→				Resolution: resolution,
   467→			})
   468→			// Add scope for each query, not just once
   469→			scopes = append(scopes, scope)
   470→		}
   471→		return queries, scopes
   472→	}
   473→
   474→	// Socket -> Socket
   475→	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeSocket {
   476→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   477→		queries = append(queries, APIQuery{
   478→			Metric:     metric,
   479→			Hostname:   hostname,
   480→			Aggregate:  false,
   481→			Type:       &socketString,
   482→			TypeIds:    intToStringSlice(sockets),
   483→			Resolution: resolution,
   484→		})
   485→		scopes = append(scopes, scope)
   486→		return queries, scopes
   487→	}
   488→
   489→	// Socket -> Node
   490→	if nativeScope == schema.MetricScopeSocket && scope == schema.MetricScopeNode {
   491→		sockets, _ := topology.GetSocketsFromHWThreads(hwthreads)
   492→		queries = append(queries, APIQuery{
   493→			Metric:     metric,
   494→			Hostname:   hostname,
   495→			Aggregate:  true,
   496→			Type:       &socketString,
   497→			TypeIds:    intToStringSlice(sockets),
   498→			Resolution: resolution,
   499→		})
   500→		scopes = append(scopes, scope)
   501→		return queries, scopes
   502→	}
   503→
   504→	// Node -> Node
   505→	if nativeScope == schema.MetricScopeNode && scope == schema.MetricScopeNode {
   506→		queries = append(queries, APIQuery{
   507→			Metric:     metric,
   508→			Hostname:   hostname,
   509→			Resolution: resolution,
   510→		})
   511→		scopes = append(scopes, scope)
   512→		return queries, scopes
   513→	}
   514→
   515→	// Unhandled Case -> Error -> Return nils
   516→	return nil, nil
   517→}
   518→
   519→// intToStringSlice converts a slice of integers to a slice of strings.
   520→// Used to convert hardware IDs (core IDs, socket IDs, etc.) to the string format required by the API.
   521→func intToStringSlice(is []int) []string {
   522→	ss := make([]string, len(is))
   523→	for i, x := range is {
   524→		ss[i] = strconv.Itoa(x)
   525→	}
   526→	return ss
   527→}
   528→


No files found


Bash

find /Users/jan/prg/ClusterCockpit/cc-backend -type d -name "metricstore" -o -type d -name "metricstoreclient"


/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore


   100→
   101→	p.pool = append(p.pool, b)
   102→}
   103→
   104→// GetSize returns the exact number of buffers currently sitting in the pool.
   105→func (p *PersistentBufferPool) GetSize() int {
   106→	p.mu.Lock()
   107→	defer p.mu.Unlock()
   108→	return len(p.pool)
   109→}
   110→
   111→// Clear drains all buffers currently in the pool, allowing the GC to collect them.
   112→func (p *PersistentBufferPool) Clear() {
   113→	p.mu.Lock()
   114→	defer p.mu.Unlock()
   115→	for i := range p.pool {
   116→		p.pool[i] = nil
   117→	}
   118→	p.pool = p.pool[:0]
   119→}
   120→
   121→// Clean removes buffers from the pool that haven't been used in the given duration.
   122→// It uses a simple LRU approach based on the lastUsed timestamp.
   123→func (p *PersistentBufferPool) Clean(threshold int64) {
   124→	p.mu.Lock()
   125→	defer p.mu.Unlock()
   126→
   127→	// Filter in place, retaining only buffers returned to the pool recently enough.
   128→	active := p.pool[:0]
   129→	for _, b := range p.pool {
   130→		if b.lastUsed >= threshold {
   131→			active = append(active, b)
   132→		}
   133→	}
   134→
   135→	// Nullify the rest to prevent memory leaks
   136→	for i := len(active); i < len(p.pool); i++ {
   137→		p.pool[i] = nil
   138→	}
   139→
   140→	p.pool = active
   141→}
   142→
   143→var (
   144→	// ErrNoData indicates no time-series data exists for the requested metric/level.
   145→	ErrNoData error = errors.New("[METRICSTORE]> no data for this metric/level")
   146→
   147→	// ErrDataDoesNotAlign indicates that aggregated data from child scopes
   148→	// does not align with the parent scope's expected timestamps/intervals.
   149→	ErrDataDoesNotAlign error = errors.New("[METRICSTORE]> data from lower granularities does not align")
   150→)
   151→
   152→// buffer stores time-series data for a single metric at a specific hierarchical level.
   153→//
   154→// Buffers form doubly-linked chains ordered by time. When capacity is reached,
   155→// a new buffer becomes the head and the old head is linked via prev/next.
   156→//
   157→// Fields:
   158→//   - prev:      Link to older buffer in the chain (nil if this is oldest)
   159→//   - next:      Link to newer buffer in the chain (nil if this is newest/head)
   160→//   - data:      Time-series values (schema.Float supports NaN for missing data)
   161→//   - frequency: Measurement interval in seconds
   162→//   - start:     Start timestamp (adjusted by -frequency/2 for alignment)
   163→//   - archived:  True if data has been persisted to disk archive
   164→//   - closed:    True if buffer is no longer accepting writes
   165→//
   166→// Index calculation: index = (timestamp - start) / frequency
   167→// Actual data timestamp: start + (frequency / 2) + (index * frequency)
   168→type buffer struct {
   169→	prev      *buffer
   170→	next      *buffer
   171→	data      []schema.Float
   172→	frequency int64
   173→	start     int64
   174→	archived  bool
   175→	closed    bool
   176→	lastUsed  int64
   177→}
   178→
   179→func newBuffer(ts, freq int64) *buffer {
   180→	b := bufferPool.Get()
   181→	b.frequency = freq
   182→	b.start = ts - (freq / 2)
   183→	b.prev = nil
   184→	b.next = nil
   185→	b.archived = false
   186→	b.closed = false
   187→	b.data = b.data[:0]
   188→	return b
   189→}
   190→
   191→// write appends a timestamped value to the buffer chain.
   192→//
   193→// Returns the head buffer (which may be newly created if capacity was reached).
   194→// Timestamps older than the buffer's start are rejected. If the calculated index
   195→// exceeds capacity, a new buffer is allocated and linked as the new head.
   196→//
   197→// Missing timestamps are automatically filled with NaN values to maintain alignment.
   198→// Overwrites are allowed if the index is already within the existing data slice.
   199→//


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides config.go: Configuration structures and metric management.
     7→//
     8→// # Configuration Hierarchy
     9→//
    10→// The metricstore package uses nested configuration structures:
    11→//
    12→//	MetricStoreConfig (Keys)
    13→//	├─ NumWorkers: Parallel checkpoint/archive workers
    14→//	├─ RetentionInMemory: How long to keep data in RAM (also used as cleanup interval)
    15→//	├─ MemoryCap: Memory limit in bytes (triggers forceFree)
    16→//	├─ Checkpoints: Persistence configuration
    17→//	│  ├─ FileFormat: "json" or "wal" (default: "wal")
    18→//	│  └─ RootDir: Checkpoint storage path
    19→//	├─ Cleanup: Long-term storage configuration (interval = RetentionInMemory)
    20→//	│  ├─ RootDir: Archive storage path (archive mode only)
    21→//	│  └─ Mode: "delete" or "archive"
    22→//	├─ Debug: Development/debugging options
    23→//	└─ Subscriptions: NATS topic subscriptions for metric ingestion
    24→//
    25→// # Metric Configuration
    26→//
    27→// Each metric (e.g., "cpu_load", "mem_used") has a MetricConfig entry in the global
    28→// Metrics map, defining:
    29→//
    30→//   - Frequency: Measurement interval in seconds
    31→//   - Aggregation: How to combine values (sum/avg/none) when transforming scopes
    32→//   - offset: Internal index into Level.metrics slice (assigned during Init)
    33→//
    34→// # AggregationStrategy
    35→//
    36→// Determines how to combine metric values when aggregating from finer to coarser scopes:
    37→//
    38→//   - NoAggregation: Do not combine (incompatible scopes)
    39→//   - SumAggregation: Add values (e.g., power consumption: core→socket)
    40→//   - AvgAggregation: Average values (e.g., temperature: core→socket)
    41→package metricstore
    42→
    43→import (
    44→	"fmt"
    45→	"time"
    46→
    47→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    48→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    49→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    50→)
    51→
    52→const (
    53→	DefaultMaxWorkers                 = 10
    54→	DefaultBufferCapacity             = 512
    55→	DefaultGCTriggerInterval          = 100
    56→	DefaultMemoryUsageTrackerInterval = 1 * time.Hour
    57→)
    58→
    59→// Checkpoints configures periodic persistence of in-memory metric data.
    60→//
    61→// Fields:
    62→//   - FileFormat: "json" (human-readable, periodic) or "wal" (binary snapshot + WAL, crash-safe); default is "wal"
    63→//   - RootDir:    Filesystem path for checkpoint files (created if missing)
    64→type Checkpoints struct {
    65→	FileFormat string `json:"file-format"`
    66→	RootDir    string `json:"directory"`
    67→}
    68→
    69→// Debug provides development and profiling options.
    70→//
    71→// Fields:
    72→//   - DumpToFile: Path to dump checkpoint data for inspection (empty = disabled)
    73→//   - EnableGops: Enable gops agent for live runtime debugging (https://github.com/google/gops)
    74→type Debug struct {
    75→	DumpToFile string `json:"dump-to-file"`
    76→	EnableGops bool   `json:"gops"`
    77→}
    78→
    79→// Cleanup configures long-term storage of old metric data.
    80→//
    81→// Data older than RetentionInMemory is archived to disk or deleted.
    82→// The cleanup interval is always RetentionInMemory.
    83→//
    84→// Fields:
    85→//   - RootDir: Filesystem path for archived data (used in "archive" mode)
    86→//   - Mode:    "delete" (discard old data) or "archive" (write to RootDir)
    87→type Cleanup struct {
    88→	RootDir string `json:"directory"`
    89→	Mode    string `json:"mode"`
    90→}
    91→
    92→// Subscriptions defines NATS topics to subscribe to for metric ingestion.
    93→//
    94→// Each subscription receives metrics via NATS messaging, enabling real-time
    95→// data collection from compute nodes.
    96→//
    97→// Fields:
    98→//   - SubscribeTo: NATS subject/channel name (e.g., "metrics.compute.*")
    99→//   - ClusterTag:  Default cluster name for metrics without cluster tag (optional)
   100→type Subscriptions []struct {
   101→	// Channel name
   102→	SubscribeTo string `json:"subscribe-to"`
   103→
   104→	// Allow lines without a cluster tag, use this as default, optional
   105→	ClusterTag string `json:"cluster-tag"`
   106→}
   107→
   108→// MetricStoreConfig defines the main configuration for the metricstore.
   109→//
   110→// Loaded from cc-backend's config.json "metricstore" section. Controls memory usage,
   111→// persistence, archiving, and metric ingestion.
   112→//
   113→// Fields:
   114→//   - NumWorkers:        Parallel workers for checkpoint/archive (0 = auto: min(NumCPU/2+1, 10))
   115→//   - RetentionInMemory: Duration string (e.g., "48h") for in-memory data retention
   116→//   - MemoryCap:         Max bytes for buffer data (0 = unlimited); triggers forceFree when exceeded
   117→//   - Checkpoints:       Periodic persistence configuration
   118→//   - Debug:             Development/profiling options (nil = disabled)
   119→//   - Archive:           Long-term storage configuration (nil = disabled)
   120→//   - Subscriptions:     NATS topics for metric ingestion (nil = polling only)
   121→type MetricStoreConfig struct {
   122→	// Number of concurrent workers for checkpoint and archive operations.
   123→	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
   124→	NumWorkers        int            `json:"num-workers"`
   125→	RetentionInMemory string         `json:"retention-in-memory"`
   126→	MemoryCap         int            `json:"memory-cap"`
   127→	Checkpoints       Checkpoints    `json:"checkpoints"`
   128→	Debug             *Debug         `json:"debug"`
   129→	Cleanup           *Cleanup       `json:"cleanup"`
   130→	Subscriptions     *Subscriptions `json:"nats-subscriptions"`
   131→}
   132→
   133→// Keys is the global metricstore configuration instance.
   134→//
   135→// Initialized with defaults, then overwritten by cc-backend's config.json.
   136→// Accessed by Init(), Checkpointing(), and other lifecycle functions.
   137→var Keys MetricStoreConfig = MetricStoreConfig{
   138→	Checkpoints: Checkpoints{
   139→		FileFormat: "wal",
   140→		RootDir:    "./var/checkpoints",
   141→	},
   142→	Cleanup: &Cleanup{
   143→		Mode: "delete",
   144→	},
   145→}
   146→
   147→// AggregationStrategy defines how to combine metric values across hierarchy levels.
   148→//
   149→// Used when transforming data from finer-grained scopes (e.g., core) to coarser scopes
   150→// (e.g., socket). This is SPATIAL aggregation, not TEMPORAL (time-based) aggregation.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package metricstore
     7→
     8→import (
     9→	"encoding/json"
    10→	"fmt"
    11→	"time"
    12→
    13→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    14→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    15→)
    16→
    17→// HealthCheckResponse represents the result of a health check operation.
    18→type HealthCheckResponse struct {
    19→	Status schema.MonitoringState
    20→	Error  error
    21→}
    22→
    23→// HealthCheckResult holds the monitoring state and raw JSON health metrics
    24→// for a single node as determined by HealthCheck.
    25→type HealthCheckResult struct {
    26→	State         schema.MonitoringState
    27→	HealthMetrics string // JSON: {"missing":[...],"degraded":[...]}
    28→}
    29→
    30→// MaxMissingDataPoints is the threshold for stale data detection.
    31→// A buffer is considered healthy if the gap between its last data point
    32→// and the current time is within MaxMissingDataPoints * frequency.
    33→const MaxMissingDataPoints int64 = 5
    34→
    35→// bufferExists returns true if the buffer is non-nil and contains data.
    36→func (b *buffer) bufferExists() bool {
    37→	if b == nil || b.data == nil || len(b.data) == 0 {
    38→		return false
    39→	}
    40→
    41→	return true
    42→}
    43→
    44→// isBufferHealthy returns true if the buffer has recent data within
    45→// MaxMissingDataPoints * frequency of the current time.
    46→func (b *buffer) isBufferHealthy() bool {
    47→	bufferEnd := b.start + b.frequency*int64(len(b.data))
    48→	t := time.Now().Unix()
    49→
    50→	return t-bufferEnd <= MaxMissingDataPoints*b.frequency
    51→}
    52→
    53→// collectMetricStatus walks the subtree rooted at l and classifies each
    54→// expected metric into the healthy or degraded map.
    55→//
    56→// Classification rules (evaluated per buffer, pessimistic):
    57→//   - A single stale buffer marks the metric as degraded permanently.
    58→//   - A healthy buffer only counts if no stale buffer has been seen.
    59→//   - Metrics absent from the global config or without any buffer remain
    60→//     in neither map and are later reported as missing.
    61→func (l *Level) collectMetricStatus(m *MemoryStore, expectedMetrics []string, healthy, degraded map[string]bool) {
    62→	l.lock.RLock()
    63→	defer l.lock.RUnlock()
    64→
    65→	for _, metricName := range expectedMetrics {
    66→		if degraded[metricName] {
    67→			continue // already degraded, cannot improve
    68→		}
    69→		mc := m.Metrics[metricName]
    70→		b := l.metrics[mc.offset]
    71→		if b.bufferExists() {
    72→			if !b.isBufferHealthy() {
    73→				degraded[metricName] = true
    74→				delete(healthy, metricName)
    75→			} else if !degraded[metricName] {
    76→				healthy[metricName] = true
    77→			}
    78→		}
    79→	}
    80→
    81→	for _, lvl := range l.children {
    82→		lvl.collectMetricStatus(m, expectedMetrics, healthy, degraded)
    83→	}
    84→}
    85→
    86→// getHealthyMetrics walks the complete subtree rooted at l and classifies
    87→// each expected metric by comparing the collected status against the
    88→// expected list.
    89→//
    90→// Returns:
    91→//   - missingList: metrics not found in global config or without any buffer
    92→//   - degradedList: metrics with at least one stale buffer in the subtree
    93→func (l *Level) getHealthyMetrics(m *MemoryStore, expectedMetrics []string) ([]string, []string) {
    94→	healthy := make(map[string]bool, len(expectedMetrics))
    95→	degraded := make(map[string]bool)
    96→
    97→	l.collectMetricStatus(m, expectedMetrics, healthy, degraded)
    98→
    99→	missingList := make([]string, 0)
   100→	degradedList := make([]string, 0)
   101→
   102→	for _, metricName := range expectedMetrics {
   103→		if healthy[metricName] {
   104→			continue
   105→		}
   106→
   107→		if degraded[metricName] {
   108→			degradedList = append(degradedList, metricName)
   109→		} else {
   110→			missingList = append(missingList, metricName)
   111→		}
   112→	}
   113→
   114→	return degradedList, missingList
   115→}
   116→
   117→// GetHealthyMetrics returns missing and degraded metric lists for a node.
   118→//
   119→// It walks the metric tree starting from the node identified by selector
   120→// and classifies each expected metric:
   121→//   - Missing: no buffer anywhere in the subtree, or metric not in global config
   122→//   - Degraded: at least one stale buffer exists in the subtree
   123→//
   124→// Metrics present in expectedMetrics but absent from both returned lists
   125→// are considered fully healthy.
   126→func (m *MemoryStore) GetHealthyMetrics(selector []string, expectedMetrics []string) ([]string, []string, error) {
   127→	lvl := m.root.findLevel(selector)
   128→	if lvl == nil {
   129→		return nil, nil, fmt.Errorf("[METRICSTORE]> GetHealthyMetrics: host not found: %#v", selector)
   130→	}
   131→
   132→	degradedList, missingList := lvl.getHealthyMetrics(m, expectedMetrics)
   133→	return degradedList, missingList, nil
   134→}
   135→
   136→type HealthCheckReq struct {
   137→	Cluster     string   `json:"cluster" example:"fritz"`
   138→	Nodes       []string `json:"nodes"`
   139→	MetricNames []string `json:"metric-names"`
   140→}
   141→
   142→// HealthCheck evaluates multiple nodes against a set of expected metrics
   143→// and returns a monitoring state per node.
   144→//
   145→// States:
   146→//   - MonitoringStateFull: all expected metrics are healthy
   147→//   - MonitoringStatePartial: some metrics are missing or degraded
   148→//   - MonitoringStateFailed: node not found, or no healthy metrics at all
   149→func (m *MemoryStore) HealthCheck(cluster string,
   150→	nodes []string, expectedMetrics []string,
   151→) (map[string]HealthCheckResult, error) {
   152→	results := make(map[string]HealthCheckResult, len(nodes))
   153→
   154→	for _, hostname := range nodes {
   155→		selector := []string{cluster, hostname}
   156→
   157→		degradedList, missingList, err := m.GetHealthyMetrics(selector, expectedMetrics)
   158→		if err != nil {
   159→			results[hostname] = HealthCheckResult{State: schema.MonitoringStateFailed}
   160→			continue
   161→		}
   162→
   163→		degradedCount := len(degradedList)
   164→		missingCount := len(missingList)
   165→
   166→		healthyCount := len(expectedMetrics) - degradedCount - missingCount
   167→
   168→		if degradedCount > 0 {
   169→			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "degraded metrics:", degradedList)
   170→		}
   171→		if missingCount > 0 {
   172→			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "missing metrics:", missingList)
   173→		}
   174→
   175→		var state schema.MonitoringState
   176→		switch {
   177→		case degradedCount == 0 && missingCount == 0:
   178→			state = schema.MonitoringStateFull
   179→		case healthyCount == 0:
   180→			state = schema.MonitoringStateFailed
   181→		default:
   182→			state = schema.MonitoringStatePartial
   183→		}
   184→
   185→		hm, _ := json.Marshal(map[string][]string{
   186→			"missing":  missingList,
   187→			"degraded": degradedList,
   188→		})
   189→
   190→		results[hostname] = HealthCheckResult{
   191→			State:         state,
   192→			HealthMetrics: string(hm),
   193→		}
   194→	}
   195→
   196→	return results, nil
   197→}
   198→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements high-level query functions for loading job metric data
     7→// with automatic scope transformation and aggregation.
     8→//
     9→// Key Concepts:
    10→//
    11→// Metric Scopes: Metrics are collected at different granularities (native scope):
    12→//   - HWThread: Per hardware thread
    13→//   - Core: Per CPU core
    14→//   - Socket: Per CPU socket
    15→//   - MemoryDomain: Per memory domain (NUMA)
    16→//   - Accelerator: Per GPU/accelerator
    17→//   - Node: Per compute node
    18→//
    19→// Scope Transformation: The buildQueries functions transform between native scope
    20→// and requested scope by:
    21→//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
    22→//   - Rejecting requests for finer granularity than available
    23→//   - Handling special cases (e.g., Accelerator metrics)
    24→//
    25→// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
    26→// based on cluster topology and job resources.
    27→package metricstore
    28→
    29→import (
    30→	"context"
    31→	"fmt"
    32→	"strconv"
    33→	"strings"
    34→	"time"
    35→
    36→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    37→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    38→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    39→)
    40→
    41→type InternalMetricStore struct{}
    42→
    43→var MetricStoreHandle *InternalMetricStore
    44→
    45→// HealthCheck delegates to the internal MemoryStore's HealthCheck.
    46→func (ccms *InternalMetricStore) HealthCheck(cluster string,
    47→	nodes []string, metrics []string,
    48→) (map[string]HealthCheckResult, error) {
    49→	return GetMemoryStore().HealthCheck(cluster, nodes, metrics)
    50→}
    51→
    52→// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
    53→// When set to a non-nil function, LoadData will call this function instead of the default implementation.
    54→var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)
    55→
    56→// LoadData loads metric data for a specific job with automatic scope transformation.
    57→//
    58→// This is the primary function for retrieving job metric data. It handles:
    59→//   - Building queries with scope transformation via buildQueries
    60→//   - Fetching data from the metric store
    61→//   - Organizing results by metric and scope
    62→//   - Converting NaN statistics to 0 for JSON compatibility
    63→//   - Partial error handling (returns data for successful queries even if some fail)
    64→//
    65→// Parameters:
    66→//   - job: Job metadata including cluster, resources, and time range
    67→//   - metrics: List of metric names to load
    68→//   - scopes: Requested metric scopes (will be transformed to match native scopes)
    69→//   - ctx: Context for cancellation (currently unused but reserved for future use)
    70→//   - resolution: Data resolution in seconds (0 for native resolution)
    71→//
    72→// Returns:
    73→//   - JobData: Map of metric → scope → JobMetric with time-series data and statistics
    74→//   - Error: Returns error if query building or fetching fails, or partial error listing failed hosts
    75→//
    76→// Example:
    77→//
    78→//	jobData, err := LoadData(job, []string{"cpu_load", "mem_used"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 60)
    79→func (ccms *InternalMetricStore) LoadData(
    80→	job *schema.Job,
    81→	metrics []string,
    82→	scopes []schema.MetricScope,
    83→	ctx context.Context,
    84→	resolution int,
    85→) (schema.JobData, error) {
    86→	if TestLoadDataCallback != nil {
    87→		return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
    88→	}
    89→
    90→	queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
    91→	if err != nil {
    92→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
    93→		return nil, err
    94→	}
    95→
    96→	// Verify assignment is correct - log any inconsistencies for debugging
    97→	if len(queries) != len(assignedScope) {
    98→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
    99→			len(queries), len(assignedScope))
   100→	}
   101→
   102→	req := APIQueryRequest{
   103→		Cluster:   job.Cluster,
   104→		From:      job.StartTime,
   105→		To:        job.StartTime + int64(job.Duration),
   106→		Queries:   queries,
   107→		WithStats: true,
   108→		WithData:  true,
   109→	}
   110→
   111→	resBody, err := FetchData(req)
   112→	if err != nil {
   113→		cclog.Errorf("Error while fetching data : %s", err.Error())
   114→		return nil, err
   115→	}
   116→
   117→	var errors []string
   118→	jobData := make(schema.JobData)
   119→
   120→	// Add safety check for potential index out of range errors
   121→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   122→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   123→			len(req.Queries), len(resBody.Results), len(assignedScope))
   124→		if len(resBody.Results) > len(req.Queries) {
   125→			resBody.Results = resBody.Results[:len(req.Queries)]
   126→		}
   127→		if len(assignedScope) > len(req.Queries) {
   128→			assignedScope = assignedScope[:len(req.Queries)]
   129→		}
   130→	}
   131→
   132→	for i, row := range resBody.Results {
   133→		// Safety check to prevent index out of range errors
   134→		if i >= len(req.Queries) || i >= len(assignedScope) {
   135→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   136→				i, len(req.Queries), len(assignedScope))
   137→			continue
   138→		}
   139→
   140→		query := req.Queries[i]
   141→		metric := query.Metric
   142→		scope := assignedScope[i]
   143→		mc := archive.GetMetricConfig(job.Cluster, metric)
   144→
   145→		if mc == nil {
   146→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   147→			continue
   148→		}
   149→
   150→		if _, ok := jobData[metric]; !ok {
   151→			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
   152→		}
   153→
   154→		res := mc.Timestep
   155→		if len(row) > 0 {
   156→			res = int(row[0].Resolution)
   157→		}
   158→
   159→		jobMetric, ok := jobData[metric][scope]
   160→		if !ok {
   161→			jobMetric = &schema.JobMetric{
   162→				Unit:     mc.Unit,
   163→				Timestep: res,
   164→				Series:   make([]schema.Series, 0),
   165→			}
   166→			jobData[metric][scope] = jobMetric
   167→		}
   168→
   169→		for ndx, res := range row {
   170→			if res.Error != nil {
   171→				/* Build list for "partial errors", if any */
   172→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   173→				continue
   174→			}
   175→
   176→			id := (*string)(nil)
   177→			if query.Type != nil {
   178→				// Check if ndx is within the bounds of TypeIds slice
   179→				if ndx < len(query.TypeIds) {
   180→					id = new(string)
   181→					*id = query.TypeIds[ndx]
   182→				} else {
   183→					// Log the error but continue processing
   184→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   185→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   186→				}
   187→			}
   188→
   189→			sanitizeStats(&res)
   190→
   191→			jobMetric.Series = append(jobMetric.Series, schema.Series{
   192→				Hostname: query.Hostname,
   193→				ID:       id,
   194→				Statistics: schema.MetricStatistics{
   195→					Avg: float64(res.Avg),
   196→					Min: float64(res.Min),
   197→					Max: float64(res.Max),
   198→				},
   199→				Data: res.Data,
   200→			})


pkg/metricstore/metricstore.go:68:type NodeProvider interface {


No matches found


Found 7 files
pkg/metricstore/query.go
internal/metricdispatch/metricdata.go
internal/metricstoreclient/cc-metric-store-queries.go
internal/metricstoreclient/cc-metric-store.go
internal/api/job.go
internal/archiver/archiveWorker.go
pkg/metricstore/api.go


   150→// (e.g., socket). This is SPATIAL aggregation, not TEMPORAL (time-based) aggregation.
   151→//
   152→// Values:
   153→//   - NoAggregation:  Do not aggregate (incompatible scopes or non-aggregatable metrics)
   154→//   - SumAggregation: Add values (e.g., power: sum core power → socket power)
   155→//   - AvgAggregation: Average values (e.g., temperature: average core temps → socket temp)
   156→type AggregationStrategy int
   157→
   158→const (
   159→	NoAggregation  AggregationStrategy = iota // Do not aggregate
   160→	SumAggregation                            // Sum values (e.g., power, energy)
   161→	AvgAggregation                            // Average values (e.g., temperature, utilization)
   162→)
   163→
   164→// AssignAggregationStrategy parses a string into an AggregationStrategy value.
   165→//
   166→// Used when loading metric configurations from JSON/YAML files.
   167→//
   168→// Parameters:
   169→//   - str: "sum", "avg", or "" (empty string for NoAggregation)
   170→//
   171→// Returns:
   172→//   - AggregationStrategy: Parsed value
   173→//   - error:               Non-nil if str is unrecognized
   174→func AssignAggregationStrategy(str string) (AggregationStrategy, error) {
   175→	switch str {
   176→	case "":
   177→		return NoAggregation, nil
   178→	case "sum":
   179→		return SumAggregation, nil
   180→	case "avg":
   181→		return AvgAggregation, nil
   182→	default:
   183→		return NoAggregation, fmt.Errorf("[METRICSTORE]> unknown aggregation strategy: %s", str)
   184→	}
   185→}
   186→
   187→// MetricConfig defines configuration for a single metric type.
   188→//
   189→// Stored in the global Metrics map, keyed by metric name (e.g., "cpu_load").
   190→//
   191→// Fields:
   192→//   - Frequency:   Measurement interval in seconds (e.g., 60 for 1-minute granularity)
   193→//   - Aggregation: How to combine values across hierarchy levels (sum/avg/none)
   194→//   - offset:      Internal index into Level.metrics slice (assigned during Init)
   195→type MetricConfig struct {
   196→	// Interval in seconds at which measurements are stored
   197→	Frequency int64
   198→
   199→	// Can be 'sum', 'avg' or null. Describes how to aggregate metrics from the same timestep over the hierarchy.
   200→	Aggregation AggregationStrategy
   201→
   202→	// Private, used internally...
   203→	offset int
   204→}
   205→
   206→func BuildMetricList() map[string]MetricConfig {
   207→	var metrics map[string]MetricConfig = make(map[string]MetricConfig)
   208→
   209→	addMetric := func(name string, metric MetricConfig) error {
   210→		if metrics == nil {
   211→			metrics = make(map[string]MetricConfig, 0)
   212→		}
   213→
   214→		if existingMetric, ok := metrics[name]; ok {
   215→			if existingMetric.Frequency != metric.Frequency {
   216→				if existingMetric.Frequency < metric.Frequency {
   217→					existingMetric.Frequency = metric.Frequency
   218→					metrics[name] = existingMetric
   219→				}
   220→			}
   221→		} else {
   222→			metrics[name] = metric
   223→		}
   224→
   225→		return nil
   226→	}
   227→
   228→	// Helper function to add metric configuration
   229→	addMetricConfig := func(mc *schema.MetricConfig) {
   230→		agg, err := AssignAggregationStrategy(mc.Aggregation)
   231→		if err != nil {
   232→			cclog.Warnf("Could not find aggregation strategy for metric config '%s': %s", mc.Name, err.Error())
   233→		}
   234→
   235→		addMetric(mc.Name, MetricConfig{
   236→			Frequency:   int64(mc.Timestep),
   237→			Aggregation: agg,
   238→		})
   239→	}
   240→	for _, c := range archive.Clusters {
   241→		for _, mc := range c.MetricConfig {
   242→			addMetricConfig(mc)
   243→		}
   244→
   245→		for _, sc := range c.SubClusters {
   246→			for _, mc := range sc.MetricConfig {
   247→				addMetricConfig(mc)
   248→			}
   249→		}


     314 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go
     277 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/archive.go
     410 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/buffer.go
     662 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/checkpoint.go
     253 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/config.go
      81 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/configSchema.go
     112 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/debug.go
     197 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go
     393 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/level.go
     366 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/lineprotocol.go
    1019 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore_test.go
     784 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go
     255 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/parquetArchive_test.go
     213 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/parquetArchive.go
    1357 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/query.go
     125 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/stats.go
     797 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go
    7615 total


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file contains the API types and data fetching logic for querying metric data
     7→// from the in-memory metric store. It provides structures for building complex queries
     8→// with support for aggregation, scaling, padding, and statistics computation.
     9→package metricstore
    10→
    11→import (
    12→	"errors"
    13→	"fmt"
    14→	"math"
    15→
    16→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    17→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    18→	"github.com/ClusterCockpit/cc-lib/v2/util"
    19→)
    20→
    21→var (
    22→	// ErrNoHostOrMetric is returned when the metric store does not find the host or the metric
    23→	ErrNoHostOrMetric error = errors.New("[METRICSTORE]> metric or host not found")
    24→	// ErrInvalidTimeRange is returned when a query has 'from' >= 'to'
    25→	ErrInvalidTimeRange = errors.New("[METRICSTORE]> invalid time range: 'from' must be before 'to'")
    26→	// ErrEmptyCluster is returned when a query with ForAllNodes has no cluster specified
    27→	ErrEmptyCluster = errors.New("[METRICSTORE]> cluster name cannot be empty")
    28→)
    29→
    30→// APIMetricData represents the response data for a single metric query.
    31→//
    32→// It contains both the time-series data points and computed statistics (avg, min, max).
    33→// If an error occurred during data retrieval, the Error field will be set and other
    34→// fields may be incomplete.
    35→type APIMetricData struct {
    36→	Error      *string           `json:"error,omitempty"`
    37→	Data       schema.FloatArray `json:"data,omitempty"`
    38→	From       int64             `json:"from"`
    39→	To         int64             `json:"to"`
    40→	Resolution int64             `json:"resolution"`
    41→	Avg        schema.Float      `json:"avg"`
    42→	Min        schema.Float      `json:"min"`
    43→	Max        schema.Float      `json:"max"`
    44→}
    45→
    46→// APIQueryRequest represents a batch query request for metric data.
    47→//
    48→// It supports two modes of operation:
    49→//  1. Explicit queries via the Queries field
    50→//  2. Automatic query generation via ForAllNodes (queries all specified metrics for all nodes in the cluster)
    51→//
    52→// The request can be customized with flags to include/exclude statistics, raw data, and padding.
    53→type APIQueryRequest struct {
    54→	Cluster     string     `json:"cluster"`
    55→	Queries     []APIQuery `json:"queries"`
    56→	ForAllNodes []string   `json:"for-all-nodes"`
    57→	From        int64      `json:"from"`
    58→	To          int64      `json:"to"`
    59→	WithStats   bool       `json:"with-stats"`
    60→	WithData    bool       `json:"with-data"`
    61→	WithPadding bool       `json:"with-padding"`
    62→}
    63→
    64→// APIQueryResponse represents the response to an APIQueryRequest.
    65→//
    66→// Results is a 2D array where each outer element corresponds to a query,
    67→// and each inner element corresponds to a selector within that query
    68→// (e.g., multiple CPUs or cores).
    69→type APIQueryResponse struct {
    70→	Queries []APIQuery        `json:"queries,omitempty"`
    71→	Results [][]APIMetricData `json:"results"`
    72→}
    73→
    74→// APIQuery represents a single metric query with optional hierarchical selectors.
    75→//
    76→// The hierarchical selection works as follows:
    77→//   - Hostname: The node to query
    78→//   - Type + TypeIds: First level of hierarchy (e.g., "cpu" + ["0", "1", "2"])
    79→//   - SubType + SubTypeIds: Second level of hierarchy (e.g., "core" + ["0", "1"])
    80→//
    81→// If Aggregate is true, data from multiple type/subtype IDs will be aggregated according
    82→// to the metric's aggregation strategy. Otherwise, separate results are returned for each combination.
    83→type APIQuery struct {
    84→	Type        *string      `json:"type,omitempty"`
    85→	SubType     *string      `json:"subtype,omitempty"`
    86→	Metric      string       `json:"metric"`
    87→	Hostname    string       `json:"host"`
    88→	Resolution  int64        `json:"resolution"`
    89→	TypeIds     []string     `json:"type-ids,omitempty"`
    90→	SubTypeIds  []string     `json:"subtype-ids,omitempty"`
    91→	ScaleFactor schema.Float `json:"scale-by,omitempty"`
    92→	Aggregate   bool         `json:"aggreg"`
    93→}
    94→
    95→// AddStats computes and populates the Avg, Min, and Max fields from the Data array.
    96→//
    97→// NaN values in the data are ignored during computation. If all values are NaN,
    98→// the statistics fields will be set to NaN.
    99→//
   100→// TODO: Optimize this, just like the stats endpoint!
   101→func (data *APIMetricData) AddStats() {
   102→	n := 0
   103→	sum, min, max := 0.0, math.MaxFloat64, -math.MaxFloat64
   104→	for _, x := range data.Data {
   105→		if x.IsNaN() {
   106→			continue
   107→		}
   108→
   109→		n += 1
   110→		sum += float64(x)
   111→		min = math.Min(min, float64(x))
   112→		max = math.Max(max, float64(x))
   113→	}
   114→
   115→	if n > 0 {
   116→		avg := sum / float64(n)
   117→		data.Avg = schema.Float(avg)
   118→		data.Min = schema.Float(min)
   119→		data.Max = schema.Float(max)
   120→	} else {
   121→		data.Avg, data.Min, data.Max = schema.NaN, schema.NaN, schema.NaN
   122→	}
   123→}
   124→
   125→// ScaleBy multiplies all data points and statistics by the given factor.
   126→//
   127→// This is commonly used for unit conversion (e.g., bytes to gigabytes).
   128→// Scaling by 0 or 1 is a no-op for performance reasons.
   129→func (data *APIMetricData) ScaleBy(f schema.Float) {
   130→	if f == 0 || f == 1 {
   131→		return
   132→	}
   133→
   134→	data.Avg *= f
   135→	data.Min *= f
   136→	data.Max *= f
   137→	for i := 0; i < len(data.Data); i++ {
   138→		data.Data[i] *= f
   139→	}
   140→}
   141→
   142→// PadDataWithNull pads the beginning of the data array with NaN values if needed.
   143→//
   144→// This ensures that the data aligns with the requested 'from' timestamp, even if
   145→// the metric store doesn't have data for the earliest time points. This is useful
   146→// for maintaining consistent array indexing across multiple queries.
   147→//
   148→// Parameters:
   149→//   - ms: MemoryStore instance to lookup metric configuration
   150→//   - from: The requested start timestamp
   151→//   - to: The requested end timestamp (unused but kept for API consistency)
   152→//   - metric: The metric name to lookup frequency information
   153→func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metric string) {
   154→	minfo, ok := ms.Metrics[metric]
   155→	if !ok {
   156→		return
   157→	}
   158→
   159→	if (data.From / minfo.Frequency) > (from / minfo.Frequency) {
   160→		padfront := int((data.From / minfo.Frequency) - (from / minfo.Frequency))
   161→		ndata := make([]schema.Float, 0, padfront+len(data.Data))
   162→		for range padfront {
   163→			ndata = append(ndata, schema.NaN)
   164→		}
   165→		for j := 0; j < len(data.Data); j++ {
   166→			ndata = append(ndata, data.Data[j])
   167→		}
   168→		data.Data = ndata
   169→	}
   170→}
   171→
   172→// FetchData executes a batch metric query request and returns the results.
   173→//
   174→// This is the primary API for retrieving metric data from the memory store. It supports:
   175→//   - Individual queries via req.Queries
   176→//   - Batch queries for all nodes via req.ForAllNodes
   177→//   - Hierarchical selector construction (cluster → host → type → subtype)
   178→//   - Optional statistics computation (avg, min, max)
   179→//   - Optional data scaling
   180→//   - Optional data padding with NaN values
   181→//
   182→// The function constructs selectors based on the query parameters and calls MemoryStore.Read()
   183→// for each selector. If a query specifies Aggregate=false with multiple type/subtype IDs,
   184→// separate results are returned for each combination.
   185→//
   186→// Parameters:
   187→//   - req: The query request containing queries, time range, and options
   188→//
   189→// Returns:
   190→//   - APIQueryResponse containing results for each query, or error if validation fails
   191→//
   192→// Errors:
   193→//   - ErrInvalidTimeRange if req.From > req.To
   194→//   - ErrEmptyCluster if req.ForAllNodes is used without specifying a cluster
   195→//   - Error if MemoryStore is not initialized
   196→//   - Individual query errors are stored in APIMetricData.Error field
   197→func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
   198→	if req.From > req.To {
   199→		return nil, ErrInvalidTimeRange
   200→	}
   201→	if req.Cluster == "" && req.ForAllNodes != nil {
   202→		return nil, ErrEmptyCluster
   203→	}
   204→
   205→	req.WithData = true
   206→	ms := GetMemoryStore()
   207→	if ms == nil {
   208→		return nil, fmt.Errorf("[METRICSTORE]> memorystore not initialized")
   209→	}
   210→
   211→	response := APIQueryResponse{
   212→		Results: make([][]APIMetricData, 0, len(req.Queries)),
   213→	}
   214→	if req.ForAllNodes != nil {
   215→		nodes := ms.ListChildren([]string{req.Cluster})
   216→		for _, node := range nodes {
   217→			for _, metric := range req.ForAllNodes {
   218→				q := APIQuery{
   219→					Metric:   metric,
   220→					Hostname: node,
   221→				}
   222→				req.Queries = append(req.Queries, q)
   223→				response.Queries = append(response.Queries, q)
   224→			}
   225→		}
   226→	}
   227→
   228→	for _, query := range req.Queries {
   229→		sels := make([]util.Selector, 0, 1)
   230→		if query.Aggregate || query.Type == nil {
   231→			sel := util.Selector{{String: req.Cluster}, {String: query.Hostname}}
   232→			if query.Type != nil {
   233→				if len(query.TypeIds) == 1 {
   234→					sel = append(sel, util.SelectorElement{String: *query.Type + query.TypeIds[0]})
   235→				} else {
   236→					ids := make([]string, len(query.TypeIds))
   237→					for i, id := range query.TypeIds {
   238→						ids[i] = *query.Type + id
   239→					}
   240→					sel = append(sel, util.SelectorElement{Group: ids})
   241→				}
   242→
   243→				if query.SubType != nil {
   244→					if len(query.SubTypeIds) == 1 {
   245→						sel = append(sel, util.SelectorElement{String: *query.SubType + query.SubTypeIds[0]})
   246→					} else {
   247→						ids := make([]string, len(query.SubTypeIds))
   248→						for i, id := range query.SubTypeIds {
   249→							ids[i] = *query.SubType + id
   250→						}
   251→						sel = append(sel, util.SelectorElement{Group: ids})
   252→					}
   253→				}
   254→			}
   255→			sels = append(sels, sel)
   256→		} else {
   257→			for _, typeID := range query.TypeIds {
   258→				if query.SubType != nil {
   259→					for _, subTypeID := range query.SubTypeIds {
   260→						sels = append(sels, util.Selector{
   261→							{String: req.Cluster},
   262→							{String: query.Hostname},
   263→							{String: *query.Type + typeID},
   264→							{String: *query.SubType + subTypeID},
   265→						})
   266→					}
   267→				} else {
   268→					sels = append(sels, util.Selector{
   269→						{String: req.Cluster},
   270→						{String: query.Hostname},
   271→						{String: *query.Type + typeID},
   272→					})
   273→				}
   274→			}
   275→		}
   276→
   277→		var err error
   278→		res := make([]APIMetricData, 0, len(sels))
   279→		for _, sel := range sels {
   280→			data := APIMetricData{}
   281→
   282→			data.Data, data.From, data.To, data.Resolution, err = ms.Read(sel, query.Metric, req.From, req.To, query.Resolution)
   283→			if err != nil {
   284→				// Skip Error If Just Missing Host or Metric, Continue
   285→				// Empty Return For Metric Handled Gracefully By Frontend
   286→				if err != ErrNoHostOrMetric {
   287→					msg := err.Error()
   288→					data.Error = &msg
   289→					res = append(res, data)
   290→				} else {
   291→					cclog.Warnf("failed to fetch '%s' from host '%s' (cluster: %s): %s", query.Metric, query.Hostname, req.Cluster, err.Error())
   292→				}
   293→				continue
   294→			}
   295→
   296→			if req.WithStats {
   297→				data.AddStats()
   298→			}
   299→			if query.ScaleFactor != 0 {
   300→				data.ScaleBy(query.ScaleFactor)
   301→			}
   302→			if req.WithPadding {
   303→				data.PadDataWithNull(ms, req.From, req.To, query.Metric)
   304→			}
   305→			if !req.WithData {
   306→				data.Data = nil
   307→			}
   308→			res = append(res, data)
   309→		}
   310→		response.Results = append(response.Results, res)
   311→	}
   312→
   313→	return &response, nil
   314→}
   315→