     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→package metricdispatch
     6→
     7→import (
     8→	"bytes"
     9→	"context"
    10→	"encoding/json"
    11→	"fmt"
    12→	"time"
    13→
    14→	"github.com/ClusterCockpit/cc-backend/internal/config"
    15→	ccms "github.com/ClusterCockpit/cc-backend/internal/metricstoreclient"
    16→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    17→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    18→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    19→)
    20→
    21→type MetricDataRepository interface {
    22→	// Return the JobData for the given job, only with the requested metrics.
    23→	LoadData(job *schema.Job,
    24→		metrics []string,
    25→		scopes []schema.MetricScope,
    26→		ctx context.Context,
    27→		resolution int) (schema.JobData, error)
    28→
    29→	// Return a map of metrics to a map of nodes to the metric statistics of the job. node scope only.
    30→	LoadStats(job *schema.Job,
    31→		metrics []string,
    32→		ctx context.Context) (map[string]map[string]schema.MetricStatistics, error)
    33→
    34→	// Return a map of metrics to a map of scopes to the scoped metric statistics of the job.
    35→	LoadScopedStats(job *schema.Job,
    36→		metrics []string,
    37→		scopes []schema.MetricScope,
    38→		ctx context.Context) (schema.ScopedJobStats, error)
    39→
    40→	// Return a map of hosts to a map of metrics at the requested scopes (currently only node) for that node.
    41→	LoadNodeData(cluster string,
    42→		metrics, nodes []string,
    43→		scopes []schema.MetricScope,
    44→		from, to time.Time,
    45→		ctx context.Context) (map[string]map[string][]*schema.JobMetric, error)
    46→
    47→	// Return a map of hosts to a map of metrics to a map of scopes for multiple nodes.
    48→	LoadNodeListData(cluster, subCluster string,
    49→		nodes []string,
    50→		metrics []string,
    51→		scopes []schema.MetricScope,
    52→		resolution int,
    53→		from, to time.Time,
    54→		ctx context.Context) (map[string]schema.JobData, error)
    55→
    56→	// HealthCheck evaluates the monitoring state for a set of nodes against expected metrics.
    57→	HealthCheck(cluster string,
    58→		nodes []string,
    59→		metrics []string) (map[string]metricstore.HealthCheckResult, error)
    60→}
    61→
    62→type CCMetricStoreConfig struct {
    63→	Scope string `json:"scope"`
    64→	URL   string `json:"url"`
    65→	Token string `json:"token"`
    66→}
    67→
    68→var metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{}
    69→
    70→func Init(rawConfig json.RawMessage) error {
    71→	if rawConfig != nil {
    72→		var configs []CCMetricStoreConfig
    73→		config.Validate(configSchema, rawConfig)
    74→		dec := json.NewDecoder(bytes.NewReader(rawConfig))
    75→		dec.DisallowUnknownFields()
    76→		if err := dec.Decode(&configs); err != nil {
    77→			return fmt.Errorf("[METRICDISPATCH]> External Metric Store Config Init: Could not decode config file '%s' Error: %s", rawConfig, err.Error())
    78→		}
    79→
    80→		if len(configs) == 0 {
    81→			return fmt.Errorf("[METRICDISPATCH]> No external metric store configurations found in config file")
    82→		}
    83→
    84→		for _, config := range configs {
    85→			metricDataRepos[config.Scope] = ccms.NewCCMetricStore(config.URL, config.Token)
    86→		}
    87→	}
    88→
    89→	return nil
    90→}
    91→
    92→func GetMetricDataRepo(cluster string, subcluster string) (MetricDataRepository, error) {
    93→	var repo MetricDataRepository
    94→	var ok bool
    95→
    96→	key := cluster + "-" + subcluster
    97→	repo, ok = metricDataRepos[key]
    98→
    99→	if !ok {
   100→		repo, ok = metricDataRepos[cluster]
   101→
   102→		if !ok {
   103→			repo, ok = metricDataRepos["*"]
   104→
   105→			if !ok {
   106→				if metricstore.MetricStoreHandle == nil {
   107→					return nil, fmt.Errorf("[METRICDISPATCH]> no metric data repository configured '%s'", key)
   108→				}
   109→
   110→				repo = metricstore.MetricStoreHandle
   111→				cclog.Debugf("[METRICDISPATCH]> Using internal metric data repository for '%s'", key)
   112→			}
   113→		}
   114→	}
   115→
   116→	return repo, nil
   117→}
   118→
   119→// GetHealthCheckRepo returns the MetricDataRepository for performing health checks on a cluster.
   120→// It uses the same fallback logic as GetMetricDataRepo: cluster → wildcard → internal.
   121→func GetHealthCheckRepo(cluster string) (MetricDataRepository, error) {
   122→	return GetMetricDataRepo(cluster, "")
   123→}
   124→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstoreclient provides a client for querying the cc-metric-store time series database.
     7→//
     8→// The cc-metric-store is a high-performance time series database optimized for HPC metric data.
     9→// This client handles HTTP communication, query construction, scope transformations, and data retrieval
    10→// for job and node metrics across different metric scopes (node, socket, core, hwthread, accelerator).
    11→//
    12→// # Architecture
    13→//
    14→// The package is split into two main components:
    15→//   - Client Operations (cc-metric-store.go): HTTP client, request handling, data loading methods
    16→//   - Query Building (cc-metric-store-queries.go): Query construction and scope transformation logic
    17→//
    18→// # Basic Usage
    19→//
    20→//	store := NewCCMetricStore("http://localhost:8080", "jwt-token")
    21→//
    22→//	// Load job data
    23→//	jobData, err := store.LoadData(job, metrics, scopes, ctx, resolution)
    24→//	if err != nil {
    25→//	    log.Fatal(err)
    26→//	}
    27→//
    28→// # Metric Scopes
    29→//
    30→// The client supports hierarchical metric scopes that map to HPC hardware topology:
    31→//   - MetricScopeAccelerator: GPU/accelerator level metrics
    32→//   - MetricScopeHWThread: Hardware thread (SMT) level metrics
    33→//   - MetricScopeCore: CPU core level metrics
    34→//   - MetricScopeSocket: CPU socket level metrics
    35→//   - MetricScopeMemoryDomain: NUMA domain level metrics
    36→//   - MetricScopeNode: Full node level metrics
    37→//
    38→// The client automatically handles scope transformations, aggregating finer-grained metrics
    39→// to coarser scopes when needed (e.g., aggregating core metrics to socket level).
    40→//
    41→// # Error Handling
    42→//
    43→// The client supports partial errors - if some queries fail, it returns both the successful
    44→// data and an error listing the failed queries. This allows processing partial results
    45→// when some nodes or metrics are temporarily unavailable.
    46→//
    47→// # API Versioning
    48→//
    49→// The client uses cc-metric-store API v2, which includes support for:
    50→//   - Data resampling for bandwidth optimization
    51→//   - Multi-scope queries in a single request
    52→//   - Aggregation across hardware topology levels
    53→package metricstoreclient
    54→
    55→import (
    56→	"bufio"
    57→	"bytes"
    58→	"context"
    59→	"encoding/json"
    60→	"fmt"
    61→	"net/http"
    62→	"strings"
    63→	"time"
    64→
    65→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    66→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    67→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    68→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    69→)
    70→
    71→// CCMetricStore is the HTTP client for communicating with cc-metric-store.
    72→// It manages connection details, authentication, and provides methods for querying metrics.
    73→type CCMetricStore struct {
    74→	client        http.Client                 // HTTP client with 10-second timeout
    75→	jwt           string                      // JWT Bearer token for authentication
    76→	url           string                      // Base URL of cc-metric-store instance
    77→	queryEndpoint string                      // Full URL to query API endpoint
    78→	topologyCache map[string]*schema.Topology // cluster -> topology cache
    79→}
    80→
    81→// APIQueryRequest represents a request to the cc-metric-store query API.
    82→// It supports both explicit queries and "for-all-nodes" bulk queries.
    83→type APIQueryRequest struct {
    84→	Cluster     string     `json:"cluster"`       // Target cluster name
    85→	Queries     []APIQuery `json:"queries"`       // Explicit list of metric queries
    86→	ForAllNodes []string   `json:"for-all-nodes"` // Metrics to query for all nodes
    87→	From        int64      `json:"from"`          // Start time (Unix timestamp)
    88→	To          int64      `json:"to"`            // End time (Unix timestamp)
    89→	WithStats   bool       `json:"with-stats"`    // Include min/avg/max statistics
    90→	WithData    bool       `json:"with-data"`     // Include time series data points
    91→}
    92→
    93→// APIQuery specifies a single metric query with optional scope filtering.
    94→// Type and TypeIds define the hardware scope (core, socket, accelerator, etc.).
    95→type APIQuery struct {
    96→	Type       *string  `json:"type,omitempty"`        // Scope type (e.g., "core", "socket")
    97→	SubType    *string  `json:"subtype,omitempty"`     // Sub-scope type (reserved for future use)
    98→	Metric     string   `json:"metric"`                // Metric name
    99→	Hostname   string   `json:"host"`                  // Target hostname
   100→	Resolution int      `json:"resolution"`            // Data resolution in seconds (0 = native)
   101→	TypeIds    []string `json:"type-ids,omitempty"`    // IDs for the scope type (e.g., core IDs)
   102→	SubTypeIds []string `json:"subtype-ids,omitempty"` // IDs for sub-scope (reserved)
   103→	Aggregate  bool     `json:"aggreg"`                // Aggregate across TypeIds
   104→}
   105→
   106→// APIQueryResponse contains the results from a cc-metric-store query.
   107→// Results align with the Queries slice by index.
   108→type APIQueryResponse struct {
   109→	Queries []APIQuery        `json:"queries,omitempty"` // Echoed queries (for bulk requests)
   110→	Results [][]APIMetricData `json:"results"`           // Result data, indexed by query
   111→}
   112→
   113→// APIMetricData represents time series data and statistics for a single metric series.
   114→// Error is set if this particular series failed to load.
   115→type APIMetricData struct {
   116→	Error      *string        `json:"error"`      // Error message if query failed
   117→	Data       []schema.Float `json:"data"`       // Time series data points
   118→	From       int64          `json:"from"`       // Actual start time of data
   119→	To         int64          `json:"to"`         // Actual end time of data
   120→	Resolution int            `json:"resolution"` // Actual resolution of data in seconds
   121→	Avg        schema.Float   `json:"avg"`        // Average value across time range
   122→	Min        schema.Float   `json:"min"`        // Minimum value in time range
   123→	Max        schema.Float   `json:"max"`        // Maximum value in time range
   124→}
   125→
   126→// NewCCMetricStore creates and initializes a new (external) CCMetricStore client.
   127→// The url parameter should include the protocol and port (e.g., "http://localhost:8080").
   128→// The token parameter is a JWT used for Bearer authentication; pass empty string if auth is disabled.
   129→func NewCCMetricStore(url string, token string) *CCMetricStore {
   130→	return &CCMetricStore{
   131→		url:           url,
   132→		queryEndpoint: fmt.Sprintf("%s/api/query", url),
   133→		jwt:           token,
   134→		client: http.Client{
   135→			Timeout: 10 * time.Second,
   136→		},
   137→		topologyCache: make(map[string]*schema.Topology),
   138→	}
   139→}
   140→
   141→// doRequest executes an HTTP POST request to the cc-metric-store query API.
   142→// It handles JSON encoding/decoding, authentication, and API versioning.
   143→// The request body is automatically closed to prevent resource leaks.
   144→func (ccms *CCMetricStore) doRequest(
   145→	ctx context.Context,
   146→	body *APIQueryRequest,
   147→) (*APIQueryResponse, error) {
   148→	buf := &bytes.Buffer{}
   149→	if err := json.NewEncoder(buf).Encode(body); err != nil {
   150→		cclog.Errorf("Error while encoding request body: %s", err.Error())
   151→		return nil, err
   152→	}
   153→
   154→	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ccms.queryEndpoint, buf)
   155→	if err != nil {
   156→		cclog.Errorf("Error while building request body: %s", err.Error())
   157→		return nil, err
   158→	}
   159→	if ccms.jwt != "" {
   160→		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   161→	}
   162→
   163→	// versioning the cc-metric-store query API.
   164→	// v2 = data with resampling
   165→	// v1 = data without resampling
   166→	q := req.URL.Query()
   167→	q.Add("version", "v2")
   168→	req.URL.RawQuery = q.Encode()
   169→
   170→	res, err := ccms.client.Do(req)
   171→	if err != nil {
   172→		cclog.Errorf("Error while performing request: %s", err.Error())
   173→		return nil, err
   174→	}
   175→	defer res.Body.Close()
   176→
   177→	if res.StatusCode != http.StatusOK {
   178→		return nil, fmt.Errorf("'%s': HTTP Status: %s", ccms.queryEndpoint, res.Status)
   179→	}
   180→
   181→	var resBody APIQueryResponse
   182→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&resBody); err != nil {
   183→		cclog.Errorf("Error while decoding result body: %s", err.Error())
   184→		return nil, err
   185→	}
   186→
   187→	return &resBody, nil
   188→}
   189→
   190→// getTopology returns the topology for a given cluster and subcluster, caching it if not already present
   191→func (ccms *CCMetricStore) getTopology(cluster, subCluster string) (*schema.Topology, error) {
   192→	cacheKey := fmt.Sprintf("%s:%s", cluster, subCluster)
   193→	if topology, ok := ccms.topologyCache[cacheKey]; ok {
   194→		return topology, nil
   195→	}
   196→
   197→	subcluster, err := archive.GetSubCluster(cluster, subCluster)
   198→	if err != nil {
   199→		return nil, err
   200→	}
   201→
   202→	ccms.topologyCache[cacheKey] = &subcluster.Topology
   203→	return &subcluster.Topology, nil
   204→}
   205→
   206→// getTopologyByNode returns the topology for a given cluster and node, caching it if not already present
   207→func (ccms *CCMetricStore) getTopologyByNode(cluster, node string) (*schema.Topology, error) {
   208→	subCluster, err := archive.GetSubClusterByNode(cluster, node)
   209→	if err != nil {
   210→		return nil, err
   211→	}
   212→
   213→	return ccms.getTopology(cluster, subCluster)
   214→}
   215→
   216→// LoadData retrieves time series data and statistics for the specified job and metrics.
   217→// It queries data for the job's time range and resources, handling scope transformations automatically.
   218→//
   219→// Parameters:
   220→//   - job: Job metadata including cluster, time range, and allocated resources
   221→//   - metrics: List of metric names to retrieve
   222→//   - scopes: Requested metric scopes (node, socket, core, etc.)
   223→//   - ctx: Context for cancellation and timeouts
   224→//   - resolution: Data resolution in seconds (0 for native resolution)
   225→//
   226→// Returns JobData organized as: metric -> scope -> series list.
   227→// Supports partial errors: returns available data even if some queries fail.
   228→func (ccms *CCMetricStore) LoadData(
   229→	job *schema.Job,
   230→	metrics []string,
   231→	scopes []schema.MetricScope,
   232→	ctx context.Context,
   233→	resolution int,
   234→) (schema.JobData, error) {
   235→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
   236→	if err != nil {
   237→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   238→		return nil, err
   239→	}
   240→
   241→	// Verify assignment is correct - log any inconsistencies for debugging
   242→	if len(queries) != len(assignedScope) {
   243→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
   244→			len(queries), len(assignedScope))
   245→	}
   246→
   247→	req := APIQueryRequest{
   248→		Cluster:   job.Cluster,
   249→		From:      job.StartTime,
   250→		To:        job.StartTime + int64(job.Duration),
   251→		Queries:   queries,
   252→		WithStats: true,
   253→		WithData:  true,
   254→	}
   255→
   256→	resBody, err := ccms.doRequest(ctx, &req)
   257→	if err != nil {
   258→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   259→		return nil, err
   260→	}
   261→
   262→	var errors []string
   263→	jobData := make(schema.JobData)
   264→
   265→	// Add safety check for potential index out of range errors
   266→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   267→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   268→			len(req.Queries), len(resBody.Results), len(assignedScope))
   269→		if len(resBody.Results) > len(req.Queries) {
   270→			resBody.Results = resBody.Results[:len(req.Queries)]
   271→		}
   272→		if len(assignedScope) > len(req.Queries) {
   273→			assignedScope = assignedScope[:len(req.Queries)]
   274→		}
   275→	}
   276→
   277→	for i, row := range resBody.Results {
   278→		// Safety check to prevent index out of range errors
   279→		if i >= len(req.Queries) || i >= len(assignedScope) {
   280→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   281→				i, len(req.Queries), len(assignedScope))
   282→			continue
   283→		}
   284→
   285→		query := req.Queries[i]
   286→		metric := query.Metric
   287→		scope := assignedScope[i]
   288→		mc := archive.GetMetricConfig(job.Cluster, metric)
   289→
   290→		if mc == nil {
   291→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   292→			continue
   293→		}
   294→
   295→		if _, ok := jobData[metric]; !ok {
   296→			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
   297→		}
   298→
   299→		res := mc.Timestep
   300→		if len(row) > 0 {
   301→			res = row[0].Resolution
   302→		}
   303→
   304→		jobMetric, ok := jobData[metric][scope]
   305→		if !ok {
   306→			jobMetric = &schema.JobMetric{
   307→				Unit:     mc.Unit,
   308→				Timestep: res,
   309→				Series:   make([]schema.Series, 0),
   310→			}
   311→			jobData[metric][scope] = jobMetric
   312→		}
   313→
   314→		for ndx, res := range row {
   315→			if res.Error != nil {
   316→				/* Build list for "partial errors", if any */
   317→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   318→				continue
   319→			}
   320→
   321→			id := (*string)(nil)
   322→			if query.Type != nil {
   323→				// Check if ndx is within the bounds of TypeIds slice
   324→				if ndx < len(query.TypeIds) {
   325→					id = new(string)
   326→					*id = query.TypeIds[ndx]
   327→				} else {
   328→					// Log the error but continue processing
   329→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   330→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   331→				}
   332→			}
   333→
   334→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   335→
   336→			jobMetric.Series = append(jobMetric.Series, schema.Series{
   337→				Hostname: query.Hostname,
   338→				ID:       id,
   339→				Statistics: schema.MetricStatistics{
   340→					Avg: float64(res.Avg),
   341→					Min: float64(res.Min),
   342→					Max: float64(res.Max),
   343→				},
   344→				Data: res.Data,
   345→			})
   346→		}
   347→
   348→		// So that one can later check len(jobData):
   349→		if len(jobMetric.Series) == 0 {
   350→			delete(jobData[metric], scope)
   351→			if len(jobData[metric]) == 0 {
   352→				delete(jobData, metric)
   353→			}
   354→		}
   355→	}
   356→
   357→	if len(errors) != 0 {
   358→		/* Returns list for "partial errors" */
   359→		return jobData, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   360→	}
   361→	return jobData, nil
   362→}
   363→
   364→// LoadStats retrieves min/avg/max statistics for job metrics at node scope.
   365→// This is faster than LoadData when only statistical summaries are needed (no time series data).
   366→//
   367→// Returns statistics organized as: metric -> hostname -> statistics.
   368→func (ccms *CCMetricStore) LoadStats(
   369→	job *schema.Job,
   370→	metrics []string,
   371→	ctx context.Context,
   372→) (map[string]map[string]schema.MetricStatistics, error) {
   373→	queries, _, err := ccms.buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0) // #166 Add scope shere for analysis view accelerator normalization?
   374→	if err != nil {
   375→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
   376→		return nil, err
   377→	}
   378→
   379→	req := APIQueryRequest{
   380→		Cluster:   job.Cluster,
   381→		From:      job.StartTime,
   382→		To:        job.StartTime + int64(job.Duration),
   383→		Queries:   queries,
   384→		WithStats: true,
   385→		WithData:  false,
   386→	}
   387→
   388→	resBody, err := ccms.doRequest(ctx, &req)
   389→	if err != nil {
   390→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   391→		return nil, err
   392→	}
   393→
   394→	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
   395→	for i, res := range resBody.Results {
   396→		if len(res) == 0 {
   397→			// No Data Found For Metric, Logged in FetchData to Warn
   398→			continue
   399→		}
   400→		query := req.Queries[i]
   401→		metric := query.Metric
   402→		data := res[0]
   403→		if data.Error != nil {
   404→			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
   405→			continue
   406→		}
   407→
   408→		metricdata, ok := stats[metric]
   409→		if !ok {
   410→			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
   411→			stats[metric] = metricdata
   412→		}
   413→
   414→		if hasNaNStats(data.Avg, data.Min, data.Max) {
   415→			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
   416→			continue
   417→		}
   418→
   419→		metricdata[query.Hostname] = schema.MetricStatistics{
   420→			Avg: float64(data.Avg),
   421→			Min: float64(data.Min),
   422→			Max: float64(data.Max),
   423→		}
   424→	}
   425→
   426→	return stats, nil
   427→}
   428→
   429→// LoadScopedStats retrieves statistics for job metrics across multiple scopes.
   430→// Used for the Job-View Statistics Table to display per-scope breakdowns.
   431→//
   432→// Returns statistics organized as: metric -> scope -> list of scoped statistics.
   433→// Each scoped statistic includes hostname, hardware ID (if applicable), and min/avg/max values.
   434→func (ccms *CCMetricStore) LoadScopedStats(
   435→	job *schema.Job,
   436→	metrics []string,
   437→	scopes []schema.MetricScope,
   438→	ctx context.Context,
   439→) (schema.ScopedJobStats, error) {
   440→	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
   441→	if err != nil {
   442→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   443→		return nil, err
   444→	}
   445→
   446→	req := APIQueryRequest{
   447→		Cluster:   job.Cluster,
   448→		From:      job.StartTime,
   449→		To:        job.StartTime + int64(job.Duration),
   450→		Queries:   queries,
   451→		WithStats: true,
   452→		WithData:  false,
   453→	}
   454→
   455→	resBody, err := ccms.doRequest(ctx, &req)
   456→	if err != nil {
   457→		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
   458→		return nil, err
   459→	}
   460→
   461→	var errors []string
   462→	scopedJobStats := make(schema.ScopedJobStats)
   463→
   464→	for i, row := range resBody.Results {
   465→		query := req.Queries[i]
   466→		metric := query.Metric
   467→		scope := assignedScope[i]
   468→
   469→		if _, ok := scopedJobStats[metric]; !ok {
   470→			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
   471→		}
   472→
   473→		if _, ok := scopedJobStats[metric][scope]; !ok {
   474→			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
   475→		}
   476→
   477→		for ndx, res := range row {
   478→			if res.Error != nil {
   479→				/* Build list for "partial errors", if any */
   480→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   481→				continue
   482→			}
   483→
   484→			id := (*string)(nil)
   485→			if query.Type != nil {
   486→				// Check if ndx is within the bounds of TypeIds slice
   487→				if ndx < len(query.TypeIds) {
   488→					id = new(string)
   489→					*id = query.TypeIds[ndx]
   490→				} else {
   491→					// Log the error but continue processing
   492→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   493→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   494→				}
   495→			}
   496→
   497→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   498→
   499→			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
   500→				Hostname: query.Hostname,
   501→				ID:       id,
   502→				Data: &schema.MetricStatistics{
   503→					Avg: float64(res.Avg),
   504→					Min: float64(res.Min),
   505→					Max: float64(res.Max),
   506→				},
   507→			})
   508→		}
   509→
   510→		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
   511→		if len(scopedJobStats[metric][scope]) == 0 {
   512→			delete(scopedJobStats[metric], scope)
   513→			if len(scopedJobStats[metric]) == 0 {
   514→				delete(scopedJobStats, metric)
   515→			}
   516→		}
   517→	}
   518→
   519→	if len(errors) != 0 {
   520→		/* Returns list for "partial errors" */
   521→		return scopedJobStats, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   522→	}
   523→	return scopedJobStats, nil
   524→}
   525→
   526→// LoadNodeData retrieves current metric data for specified nodes in a cluster.
   527→// Used for the Systems-View Node-Overview to display real-time node status.
   528→//
   529→// If nodes is nil, queries all metrics for all nodes in the cluster (bulk query).
   530→// Returns data organized as: hostname -> metric -> list of JobMetric (with time series and stats).
   531→func (ccms *CCMetricStore) LoadNodeData(
   532→	cluster string,
   533→	metrics, nodes []string,
   534→	scopes []schema.MetricScope,
   535→	from, to time.Time,
   536→	ctx context.Context,
   537→) (map[string]map[string][]*schema.JobMetric, error) {
   538→	req := APIQueryRequest{
   539→		Cluster:   cluster,
   540→		From:      from.Unix(),
   541→		To:        to.Unix(),
   542→		WithStats: true,
   543→		WithData:  true,
   544→	}
   545→
   546→	if nodes == nil {
   547→		req.ForAllNodes = append(req.ForAllNodes, metrics...)
   548→	} else {
   549→		for _, node := range nodes {
   550→			for _, metric := range metrics {
   551→				req.Queries = append(req.Queries, APIQuery{
   552→					Hostname:   node,
   553→					Metric:     metric,
   554→					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
   555→				})
   556→			}
   557→		}
   558→	}
   559→
   560→	resBody, err := ccms.doRequest(ctx, &req)
   561→	if err != nil {
   562→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   563→		return nil, err
   564→	}
   565→
   566→	var errors []string
   567→	data := make(map[string]map[string][]*schema.JobMetric)
   568→	for i, res := range resBody.Results {
   569→		if len(res) == 0 {
   570→			// No Data Found For Metric, Logged in FetchData to Warn
   571→			continue
   572→		}
   573→
   574→		var query APIQuery
   575→		if resBody.Queries != nil {
   576→			query = resBody.Queries[i]
   577→		} else {
   578→			query = req.Queries[i]
   579→		}
   580→
   581→		metric := query.Metric
   582→		qdata := res[0]
   583→		if qdata.Error != nil {
   584→			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
   585→		}
   586→
   587→		sanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)
   588→
   589→		hostdata, ok := data[query.Hostname]
   590→		if !ok {
   591→			hostdata = make(map[string][]*schema.JobMetric)
   592→			data[query.Hostname] = hostdata
   593→		}
   594→
   595→		mc := archive.GetMetricConfig(cluster, metric)
   596→		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
   597→			Unit:     mc.Unit,
   598→			Timestep: mc.Timestep,
   599→			Series: []schema.Series{
   600→				{
   601→					Hostname: query.Hostname,
   602→					Data:     qdata.Data,
   603→					Statistics: schema.MetricStatistics{
   604→						Avg: float64(qdata.Avg),
   605→						Min: float64(qdata.Min),
   606→						Max: float64(qdata.Max),
   607→					},
   608→				},
   609→			},
   610→		})
   611→	}
   612→
   613→	if len(errors) != 0 {
   614→		/* Returns list of "partial errors" */
   615→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   616→	}
   617→
   618→	return data, nil
   619→}
   620→
   621→// LoadNodeListData retrieves paginated node metrics for the Systems-View Node-List.
   622→//
   623→// Supports filtering by subcluster and node name pattern. The nodeFilter performs
   624→// substring matching on hostnames.
   625→//
   626→// Returns:
   627→//   - Node data organized as: hostname -> JobData (metric -> scope -> series)
   628→//   - Total node count (before pagination)
   629→//   - HasNextPage flag indicating if more pages are available
   630→//   - Error (may be partial error with some data returned)
   631→func (ccms *CCMetricStore) LoadNodeListData(
   632→	cluster, subCluster string,
   633→	nodes []string,
   634→	metrics []string,
   635→	scopes []schema.MetricScope,
   636→	resolution int,
   637→	from, to time.Time,
   638→	ctx context.Context,
   639→) (map[string]schema.JobData, error) {
   640→	queries, assignedScope, err := ccms.buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, resolution)
   641→	if err != nil {
   642→		cclog.Errorf("Error while building node queries for Cluster %s, SubCluster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
   643→		return nil, err
   644→	}
   645→
   646→	// Verify assignment is correct - log any inconsistencies for debugging
   647→	if len(queries) != len(assignedScope) {
   648→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
   649→			len(queries), len(assignedScope))
   650→	}
   651→
   652→	req := APIQueryRequest{
   653→		Cluster:   cluster,
   654→		Queries:   queries,
   655→		From:      from.Unix(),
   656→		To:        to.Unix(),
   657→		WithStats: true,
   658→		WithData:  true,
   659→	}
   660→
   661→	resBody, err := ccms.doRequest(ctx, &req)
   662→	if err != nil {
   663→		cclog.Errorf("Error while performing request for cluster %s: %s", cluster, err.Error())
   664→		return nil, err
   665→	}
   666→
   667→	var errors []string
   668→	data := make(map[string]schema.JobData)
   669→
   670→	// Add safety check for index out of range issues
   671→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   672→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   673→			len(req.Queries), len(resBody.Results), len(assignedScope))
   674→		if len(resBody.Results) > len(req.Queries) {
   675→			resBody.Results = resBody.Results[:len(req.Queries)]
   676→		}
   677→		if len(assignedScope) > len(req.Queries) {
   678→			assignedScope = assignedScope[:len(req.Queries)]
   679→		}
   680→	}
   681→
   682→	for i, row := range resBody.Results {
   683→		// Safety check to prevent index out of range errors
   684→		if i >= len(req.Queries) || i >= len(assignedScope) {
   685→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   686→				i, len(req.Queries), len(assignedScope))
   687→			continue
   688→		}
   689→
   690→		var query APIQuery
   691→		if resBody.Queries != nil {
   692→			if i < len(resBody.Queries) {
   693→				query = resBody.Queries[i]
   694→			} else {
   695→				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
   696→					i, len(resBody.Queries))
   697→				continue
   698→			}
   699→		} else {
   700→			query = req.Queries[i]
   701→		}
   702→
   703→		metric := query.Metric
   704→		scope := assignedScope[i]
   705→		mc := archive.GetMetricConfig(cluster, metric)
   706→		if mc == nil {
   707→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   708→			continue
   709→		}
   710→
   711→		res := mc.Timestep
   712→		if len(row) > 0 {
   713→			res = row[0].Resolution
   714→		}
   715→
   716→		// Init Nested Map Data Structures If Not Found
   717→		hostData, ok := data[query.Hostname]
   718→		if !ok {
   719→			hostData = make(schema.JobData)
   720→			data[query.Hostname] = hostData
   721→		}
   722→
   723→		metricData, ok := hostData[metric]
   724→		if !ok {
   725→			metricData = make(map[schema.MetricScope]*schema.JobMetric)
   726→			data[query.Hostname][metric] = metricData
   727→		}
   728→
   729→		scopeData, ok := metricData[scope]
   730→		if !ok {
   731→			scopeData = &schema.JobMetric{
   732→				Unit:     mc.Unit,
   733→				Timestep: res,
   734→				Series:   make([]schema.Series, 0),
   735→			}
   736→			data[query.Hostname][metric][scope] = scopeData
   737→		}
   738→
   739→		for ndx, res := range row {
   740→			if res.Error != nil {
   741→				/* Build list for "partial errors", if any */
   742→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   743→				continue
   744→			}
   745→
   746→			id := (*string)(nil)
   747→			if query.Type != nil {
   748→				// Check if ndx is within the bounds of TypeIds slice
   749→				if ndx < len(query.TypeIds) {
   750→					id = new(string)
   751→					*id = query.TypeIds[ndx]
   752→				} else {
   753→					// Log the error but continue processing
   754→					cclog.Warnf("TypeIds index out of range: %d with length %d for metric %s on host %s",
   755→						ndx, len(query.TypeIds), query.Metric, query.Hostname)
   756→				}
   757→			}
   758→
   759→			sanitizeStats(&res.Avg, &res.Min, &res.Max)
   760→
   761→			scopeData.Series = append(scopeData.Series, schema.Series{
   762→				Hostname: query.Hostname,
   763→				ID:       id,
   764→				Statistics: schema.MetricStatistics{
   765→					Avg: float64(res.Avg),
   766→					Min: float64(res.Min),
   767→					Max: float64(res.Max),
   768→				},
   769→				Data: res.Data,
   770→			})
   771→		}
   772→	}
   773→
   774→	if len(errors) != 0 {
   775→		/* Returns list of "partial errors" */
   776→		return data, fmt.Errorf("METRICDATA/EXTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   777→	}
   778→
   779→	return data, nil
   780→}
   781→
   782→// HealthCheck queries the external cc-metric-store's health check endpoint.
   783→// It sends a HealthCheckReq as the request body to /api/healthcheck and
   784→// returns the per-node health check results.
   785→func (ccms *CCMetricStore) HealthCheck(cluster string,
   786→	nodes []string, metrics []string,
   787→) (map[string]metricstore.HealthCheckResult, error) {
   788→	req := metricstore.HealthCheckReq{
   789→		Cluster:     cluster,
   790→		Nodes:       nodes,
   791→		MetricNames: metrics,
   792→	}
   793→
   794→	buf := &bytes.Buffer{}
   795→	if err := json.NewEncoder(buf).Encode(req); err != nil {
   796→		cclog.Errorf("Error while encoding health check request body: %s", err.Error())
   797→		return nil, err
   798→	}
   799→
   800→	endpoint := fmt.Sprintf("%s/api/healthcheck", ccms.url)
   801→	httpReq, err := http.NewRequest(http.MethodGet, endpoint, buf)
   802→	if err != nil {
   803→		cclog.Errorf("Error while building health check request: %s", err.Error())
   804→		return nil, err
   805→	}
   806→	if ccms.jwt != "" {
   807→		httpReq.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ccms.jwt))
   808→	}
   809→
   810→	res, err := ccms.client.Do(httpReq)
   811→	if err != nil {
   812→		cclog.Errorf("Error while performing health check request: %s", err.Error())
   813→		return nil, err
   814→	}
   815→	defer res.Body.Close()
   816→
   817→	if res.StatusCode != http.StatusOK {
   818→		return nil, fmt.Errorf("'%s': HTTP Status: %s", endpoint, res.Status)
   819→	}
   820→
   821→	var results map[string]metricstore.HealthCheckResult
   822→	if err := json.NewDecoder(bufio.NewReader(res.Body)).Decode(&results); err != nil {
   823→		cclog.Errorf("Error while decoding health check response: %s", err.Error())
   824→		return nil, err
   825→	}
   826→
   827→	return results, nil
   828→}
   829→
   830→// sanitizeStats replaces NaN values in statistics with 0 to enable JSON marshaling.
   831→// Regular float64 values cannot be JSONed when NaN.
   832→func sanitizeStats(avg, min, max *schema.Float) {
   833→	if avg.IsNaN() || min.IsNaN() || max.IsNaN() {
   834→		*avg = schema.Float(0)
   835→		*min = schema.Float(0)
   836→		*max = schema.Float(0)
   837→	}
   838→}
   839→
   840→// hasNaNStats returns true if any of the statistics contain NaN values.
   841→func hasNaNStats(avg, min, max schema.Float) bool {
   842→	return avg.IsNaN() || min.IsNaN() || max.IsNaN()
   843→}
   844→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements ingestion of InfluxDB line-protocol metric data received
     7→// over NATS. Each line encodes one metric sample with the following structure:
     8→//
     9→//	<measurement>[,cluster=<c>][,hostname=<h>][,type=<t>][,type-id=<id>][,subtype=<s>][,stype-id=<id>] value=<v> [<timestamp>]
    10→//
    11→// The measurement name identifies the metric (e.g. "cpu_load"). Tags provide
    12→// routing information (cluster, host) and optional sub-device selectors (type,
    13→// subtype). Only one field is expected per line: "value".
    14→//
    15→// After decoding, each sample is:
    16→//  1. Written to the in-memory store via ms.WriteToLevel.
    17→//  2. If the checkpoint format is "wal", also forwarded to the WAL staging
    18→//     goroutine via the WALMessages channel for durable write-ahead logging.
    19→package metricstore
    20→
    21→import (
    22→	"bytes"
    23→	"context"
    24→	"fmt"
    25→	"sync"
    26→	"time"
    27→
    28→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    29→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    30→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    31→	"github.com/ClusterCockpit/cc-line-protocol/v2/lineprotocol"
    32→)
    33→
    34→// ReceiveNats subscribes to all configured NATS subjects and feeds incoming
    35→// line-protocol messages into the MemoryStore.
    36→//
    37→// When workers > 1 a pool of goroutines drains a shared channel so that
    38→// multiple messages can be decoded in parallel. With workers == 1 the NATS
    39→// callback decodes inline (no channel overhead, lower latency).
    40→//
    41→// The function blocks until ctx is cancelled and all worker goroutines have
    42→// finished. It returns nil when the NATS client is not configured; callers
    43→// should treat that as a no-op rather than an error.
    44→func ReceiveNats(ms *MemoryStore,
    45→	workers int,
    46→	ctx context.Context,
    47→) error {
    48→	nc := nats.GetClient()
    49→
    50→	if nc == nil {
    51→		cclog.Warn("NATS client not initialized")
    52→		return nil
    53→	}
    54→
    55→	var wg sync.WaitGroup
    56→	msgs := make(chan []byte, workers*2)
    57→
    58→	for _, sc := range *Keys.Subscriptions {
    59→		clusterTag := sc.ClusterTag
    60→		if workers > 1 {
    61→			wg.Add(workers)
    62→
    63→			for range workers {
    64→				go func() {
    65→					defer wg.Done()
    66→					for m := range msgs {
    67→						dec := lineprotocol.NewDecoderWithBytes(m)
    68→						if err := DecodeLine(dec, ms, clusterTag); err != nil {
    69→							cclog.Errorf("error: %s", err.Error())
    70→						}
    71→					}
    72→				}()
    73→			}
    74→
    75→			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
    76→				select {
    77→				case msgs <- data:
    78→				case <-ctx.Done():
    79→				}
    80→			})
    81→		} else {
    82→			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
    83→				dec := lineprotocol.NewDecoderWithBytes(data)
    84→				if err := DecodeLine(dec, ms, clusterTag); err != nil {
    85→					cclog.Errorf("error: %s", err.Error())
    86→				}
    87→			})
    88→		}
    89→		cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo)
    90→	}
    91→
    92→	go func() {
    93→		<-ctx.Done()
    94→		close(msgs)
    95→	}()
    96→
    97→	wg.Wait()
    98→
    99→	return nil
   100→}


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements checkpoint persistence for the in-memory metric store.
     7→//
     8→// Checkpoints enable graceful restarts by periodically saving in-memory metric
     9→// data to disk in JSON or binary format. The checkpoint system:
    10→//
    11→// Key Features:
    12→//   - Periodic background checkpointing via the Checkpointing() worker
    13→//   - Two format families: JSON (human-readable) and WAL+binary (compact, crash-safe)
    14→//   - Parallel checkpoint creation and loading using worker pools
    15→//   - Hierarchical file organization: checkpoint_dir/cluster/host/timestamp.{json|bin}
    16→//   - WAL file: checkpoint_dir/cluster/host/current.wal (append-only, per-entry)
    17→//   - Only saves unarchived data (archived data is already persisted elsewhere)
    18→//   - GC optimization during loading to prevent excessive heap growth
    19→//
    20→// Checkpoint Workflow:
    21→//  1. Init() loads checkpoints within retention window at startup
    22→//  2. Checkpointing() worker periodically saves new data
    23→//  3. Shutdown() writes final checkpoint before exit
    24→//
    25→// File Organization:
    26→//
    27→//	checkpoints/
    28→//	  cluster1/
    29→//	    host001/
    30→//	      1234567890.json  (JSON format: full subtree snapshot)
    31→//	      1234567890.bin   (binary format: full subtree snapshot)
    32→//	      current.wal      (WAL format: append-only per-entry log)
    33→//	    host002/
    34→//	      ...
    35→package metricstore
    36→
    37→import (
    38→	"bufio"
    39→	"context"
    40→	"encoding/json"
    41→	"errors"
    42→	"fmt"
    43→	"io/fs"
    44→	"os"
    45→	"path"
    46→	"path/filepath"
    47→	"sort"
    48→	"strconv"
    49→	"strings"
    50→	"sync"
    51→	"sync/atomic"
    52→	"time"
    53→
    54→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    55→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    56→)
    57→
    58→const (
    59→	CheckpointFilePerms = 0o644                    // File permissions for checkpoint files
    60→	CheckpointDirPerms  = 0o755                    // Directory permissions for checkpoint directories
    61→	GCTriggerInterval   = DefaultGCTriggerInterval // Interval for triggering GC during checkpoint loading
    62→)
    63→
    64→// CheckpointMetrics represents metric data in a checkpoint file.
    65→// Whenever the structure changes, update MarshalJSON as well!
    66→type CheckpointMetrics struct {
    67→	Data      []schema.Float `json:"data"`
    68→	Frequency int64          `json:"frequency"`
    69→	Start     int64          `json:"start"`
    70→}
    71→
    72→// CheckpointFile represents the hierarchical structure of a checkpoint file.
    73→// It mirrors the Level tree structure from the MemoryStore.
    74→type CheckpointFile struct {
    75→	Metrics  map[string]*CheckpointMetrics `json:"metrics"`
    76→	Children map[string]*CheckpointFile    `json:"children"`
    77→	From     int64                         `json:"from"`
    78→	To       int64                         `json:"to"`
    79→}
    80→


pkg/metricstore/configSchema.go:8:const configSchema = `{
pkg/metricstore/parquetArchive_test.go:18:func TestParseScopeFromName(t *testing.T) {
pkg/metricstore/parquetArchive_test.go:47:func TestFlattenCheckpointFile(t *testing.T) {
pkg/metricstore/parquetArchive_test.go:106:func TestParquetArchiveRoundtrip(t *testing.T) {
pkg/metricstore/parquetArchive_test.go:214:func TestLoadCheckpointFileFromDisk_JSON(t *testing.T) {
pkg/metricstore/metricstore_test.go:19:func TestBufferPoolGetReuse(t *testing.T) {
pkg/metricstore/metricstore_test.go:44:func TestBufferPoolClear(t *testing.T) {
pkg/metricstore/metricstore_test.go:58:func TestBufferEndFirstWrite(t *testing.T) {
pkg/metricstore/metricstore_test.go:73:func TestBufferWriteNaNFill(t *testing.T) {
pkg/metricstore/metricstore_test.go:98:func TestBufferWriteCapacityOverflow(t *testing.T) {
pkg/metricstore/metricstore_test.go:127:func TestBufferWriteOverwrite(t *testing.T) {
pkg/metricstore/metricstore_test.go:146:func TestBufferReadBeforeFirstWrite(t *testing.T) {
pkg/metricstore/metricstore_test.go:169:func TestBufferReadChain(t *testing.T) {
pkg/metricstore/metricstore_test.go:203:func TestBufferReadIdxAfterSwitch(t *testing.T) {
pkg/metricstore/metricstore_test.go:243:func TestBufferReadNaNValues(t *testing.T) {
pkg/metricstore/metricstore_test.go:270:func TestBufferReadAccumulation(t *testing.T) {
pkg/metricstore/metricstore_test.go:294:func newTestPool(t *testing.T) *PersistentBufferPool {
pkg/metricstore/metricstore_test.go:305:func TestBufferFreeRetention(t *testing.T) {
pkg/metricstore/metricstore_test.go:343:func TestBufferFreeAll(t *testing.T) {
pkg/metricstore/metricstore_test.go:372:func TestForceFreeOldestPoolReturn(t *testing.T) {
pkg/metricstore/metricstore_test.go:406:func TestForceFreeOldestSingleBuffer(t *testing.T) {
pkg/metricstore/metricstore_test.go:423:func TestBufferIterFromToOrder(t *testing.T) {
pkg/metricstore/metricstore_test.go:451:func TestBufferIterFromToFiltered(t *testing.T) {
pkg/metricstore/metricstore_test.go:477:func TestBufferIterFromToNilBuffer(t *testing.T) {
pkg/metricstore/metricstore_test.go:496:func TestBufferCount(t *testing.T) {
pkg/metricstore/metricstore_test.go:518:func TestAssignAggregationStrategy(t *testing.T) {
pkg/metricstore/metricstore_test.go:545:func TestBufferWrite(t *testing.T) {
pkg/metricstore/metricstore_test.go:570:func TestBufferRead(t *testing.T) {
pkg/metricstore/metricstore_test.go:596:func TestHealthCheck(t *testing.T) {
pkg/metricstore/metricstore_test.go:769:func TestGetHealthyMetrics(t *testing.T) {
pkg/metricstore/metricstore_test.go:878:func TestBufferHealthChecks(t *testing.T) {
pkg/metricstore/metricstore_test.go:971:func TestBufferPoolClean(t *testing.T) {
pkg/metricstore/lineprotocol.go:44:func ReceiveNats(ms *MemoryStore,
pkg/metricstore/lineprotocol.go:109:func reorder(buf, prefix []byte) []byte {
pkg/metricstore/lineprotocol.go:129:type decodeState struct {
pkg/metricstore/lineprotocol.go:161:var decodeStatePool = sync.Pool{
pkg/metricstore/lineprotocol.go:194:func DecodeLine(dec *lineprotocol.Decoder,
pkg/metricstore/api.go:21:var (
pkg/metricstore/api.go:35:type APIMetricData struct {
pkg/metricstore/api.go:53:type APIQueryRequest struct {
pkg/metricstore/api.go:69:type APIQueryResponse struct {
pkg/metricstore/api.go:83:type APIQuery struct {
pkg/metricstore/api.go:101:func (data *APIMetricData) AddStats() {
pkg/metricstore/api.go:129:func (data *APIMetricData) ScaleBy(f schema.Float) {
pkg/metricstore/api.go:153:func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metric string) {
pkg/metricstore/api.go:197:func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
pkg/metricstore/level.go:62:type Level struct {
pkg/metricstore/level.go:86:func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
pkg/metricstore/level.go:148:func (l *Level) collectPaths(currentDepth, targetDepth int, currentPath []string, results *[][]string) {
pkg/metricstore/level.go:183:func (l *Level) free(t int64) (int, error) {
pkg/metricstore/level.go:225:func (l *Level) forceFree() (int, error) {
pkg/metricstore/level.go:273:func (l *Level) sizeInBytes() int64 {
pkg/metricstore/level.go:301:func (l *Level) findLevel(selector []string) *Level {
pkg/metricstore/level.go:339:func (l *Level) findBuffers(selector util.Selector, offset int, f func(b *buffer) error) error {
pkg/metricstore/debug.go:14:func (b *buffer) debugDump(buf []byte) []byte {
pkg/metricstore/debug.go:37:func (l *Level) debugDump(m *MemoryStore, w *bufio.Writer, lvlname string, buf []byte, depth int) ([]byte, error) {
pkg/metricstore/debug.go:92:func (m *MemoryStore) DebugDump(w *bufio.Writer, selector []string) error {
pkg/metricstore/buffer.go:55:const BufferCap int = DefaultBufferCapacity
pkg/metricstore/buffer.go:59:var bufferPool = NewPersistentBufferPool()
pkg/metricstore/buffer.go:61:type PersistentBufferPool struct {
pkg/metricstore/buffer.go:67:func NewPersistentBufferPool() *PersistentBufferPool {
pkg/metricstore/buffer.go:73:func (p *PersistentBufferPool) Get() *buffer {
pkg/metricstore/buffer.go:94:func (p *PersistentBufferPool) Put(b *buffer) {
pkg/metricstore/buffer.go:105:func (p *PersistentBufferPool) GetSize() int {
pkg/metricstore/buffer.go:112:func (p *PersistentBufferPool) Clear() {
pkg/metricstore/buffer.go:123:func (p *PersistentBufferPool) Clean(threshold int64) {
pkg/metricstore/buffer.go:143:var (
pkg/metricstore/buffer.go:168:type buffer struct {
pkg/metricstore/buffer.go:179:func newBuffer(ts, freq int64) *buffer {
pkg/metricstore/buffer.go:207:func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
pkg/metricstore/buffer.go:237:func (b *buffer) end() int64 {
pkg/metricstore/buffer.go:241:func (b *buffer) firstWrite() int64 {
pkg/metricstore/buffer.go:266:func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int64, int64, error) {
pkg/metricstore/buffer.go:317:func (b *buffer) free(t int64) (delme bool, n int) {
pkg/metricstore/buffer.go:346:func (b *buffer) forceFreeOldest() (delme bool, n int) {
pkg/metricstore/buffer.go:382:func (b *buffer) iterFromTo(from, to int64, callback func(b *buffer) error) error {
pkg/metricstore/buffer.go:404:func (b *buffer) count() int64 {
pkg/metricstore/stats.go:16:type Stats struct {
pkg/metricstore/stats.go:23:func (b *buffer) stats(from, to int64) (Stats, int64, int64, error) {
pkg/metricstore/stats.go:74:func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int64) (*Stats, int64, int64, error) {
pkg/metricstore/metricstore.go:42:type GlobalState struct {
pkg/metricstore/metricstore.go:48:var (
pkg/metricstore/metricstore.go:68:type NodeProvider interface {
pkg/metricstore/metricstore.go:82:type Metric struct {
pkg/metricstore/metricstore.go:98:type MemoryStore struct {
pkg/metricstore/metricstore.go:124:func Init(rawConfig json.RawMessage, metrics map[string]MetricConfig, wg *sync.WaitGroup) {
pkg/metricstore/metricstore.go:205:func InitMetrics(metrics map[string]MetricConfig) {
pkg/metricstore/metricstore.go:236:func GetMemoryStore() *MemoryStore {
pkg/metricstore/metricstore.go:244:func (ms *MemoryStore) GetMetricFrequency(metricName string) (int64, error) {
pkg/metricstore/metricstore.go:255:func (ms *MemoryStore) SetNodeProvider(provider NodeProvider) {
pkg/metricstore/metricstore.go:272:func Shutdown() {
pkg/metricstore/metricstore.go:320:func Retention(wg *sync.WaitGroup, ctx context.Context) {
pkg/metricstore/metricstore.go:383:func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
pkg/metricstore/metricstore.go:496:func Free(ms *MemoryStore, t time.Time) (int, error) {
pkg/metricstore/metricstore.go:535:func FreeSelected(ms *MemoryStore, selectors [][]string, t time.Time) (int, error) {
pkg/metricstore/metricstore.go:567:func GetSelectors(ms *MemoryStore, excludeSelectors map[string][]string) [][]string {
pkg/metricstore/metricstore.go:599:func (ms *MemoryStore) GetPaths(targetDepth int) [][]string {
pkg/metricstore/metricstore.go:611:func (m *MemoryStore) Write(selector []string, ts int64, metrics []Metric) error {
pkg/metricstore/metricstore.go:627:func (m *MemoryStore) GetLevel(selector []string) *Level {
pkg/metricstore/metricstore.go:632:func (m *MemoryStore) WriteToLevel(l *Level, selector []string, ts int64, metrics []Metric) error {

[Showing results with pagination = limit: 100, offset: 0]


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package metricstore provides an efficient in-memory time-series metric storage system
     7→// with support for hierarchical data organization, checkpointing, and archiving.
     8→//
     9→// The package organizes metrics in a tree structure (cluster → host → component) and
    10→// provides concurrent read/write access to metric data with configurable aggregation strategies.
    11→// Background goroutines handle periodic checkpointing (JSON or Avro format), archiving old data,
    12→// and enforcing retention policies.
    13→//
    14→// Key features:
    15→//   - In-memory metric storage with configurable retention
    16→//   - Hierarchical data organization (selectors)
    17→//   - Concurrent checkpoint/archive workers
    18→//   - Support for sum and average aggregation
    19→//   - NATS integration for metric ingestion
    20→package metricstore
    21→
    22→import (
    23→	"bytes"
    24→	"context"
    25→	"encoding/json"
    26→	"errors"
    27→	"fmt"
    28→	"runtime"
    29→	"runtime/debug"
    30→	"slices"
    31→	"sync"
    32→	"time"
    33→
    34→	"github.com/ClusterCockpit/cc-backend/internal/config"
    35→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    36→	"github.com/ClusterCockpit/cc-lib/v2/resampler"
    37→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    38→	"github.com/ClusterCockpit/cc-lib/v2/util"
    39→)
    40→
    41→// GlobalState holds the global state for the metric store with thread-safe access.
    42→type GlobalState struct {
    43→	mu                sync.RWMutex
    44→	lastRetentionTime int64
    45→	selectorsExcluded bool
    46→}
    47→
    48→var (
    49→	singleton  sync.Once
    50→	msInstance *MemoryStore
    51→	// shutdownFunc stores the context cancellation function created in Init
    52→	// and is called during Shutdown to cancel all background goroutines
    53→	shutdownFunc   context.CancelFunc
    54→	shutdownFuncMu sync.Mutex // Protects shutdownFunc from concurrent access
    55→	// Create a global instance
    56→	state = &GlobalState{}
    57→)
    58→
    59→// NodeProvider provides information about nodes currently in use by running jobs.
    60→//
    61→// This interface allows metricstore to query job information without directly
    62→// depending on the repository package, breaking the import cycle.
    63→//
    64→// Implementations should return nodes that are actively processing jobs started
    65→// before the given timestamp. These nodes will be excluded from retention-based
    66→// garbage collection to prevent data loss for jobs that are still running or
    67→// recently completed.
    68→type NodeProvider interface {
    69→	// GetUsedNodes returns a map of cluster names to sorted lists of unique hostnames
    70→	// that are currently in use by jobs that started before the given timestamp.
    71→	//
    72→	// Parameters:
    73→	//   - ts: Unix timestamp threshold - returns nodes with jobs started before this time
    74→	//
    75→	// Returns:
    76→	//   - Map of cluster names to lists of node hostnames that should be excluded from garbage collection
    77→	//   - Error if the query fails
    78→	GetUsedNodes(ts int64) (map[string][]string, error)
    79→}
    80→
    81→// Metric represents a single metric data point to be written to the store.
    82→type Metric struct {
    83→	Name  string
    84→	Value schema.Float
    85→	// MetricConfig contains frequency and aggregation settings for this metric.
    86→	// If Frequency is 0, configuration will be looked up from MemoryStore.Metrics during Write().
    87→	MetricConfig MetricConfig
    88→}
    89→
    90→// MemoryStore is the main in-memory time-series metric storage implementation.
    91→//
    92→// It organizes metrics in a hierarchical tree structure where each level represents
    93→// a component of the system hierarchy (e.g., cluster → host → CPU). Each level can
    94→// store multiple metrics as time-series buffers.
    95→//
    96→// The store is initialized as a singleton via InitMetrics() and accessed via GetMemoryStore().
    97→// All public methods are safe for concurrent use.
    98→type MemoryStore struct {
    99→	Metrics      map[string]MetricConfig
   100→	root         Level


Bash

grep -n "HealthCheck" /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/*.go


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:17:// HealthCheckResponse represents the result of a health check operation.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:18:type HealthCheckResponse struct {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:23:// HealthCheckResult holds the monitoring state and raw JSON health metrics
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:24:// for a single node as determined by HealthCheck.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:25:type HealthCheckResult struct {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:136:type HealthCheckReq struct {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:142:// HealthCheck evaluates multiple nodes against a set of expected metrics
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:149:func (m *MemoryStore) HealthCheck(cluster string,
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:151:) (map[string]HealthCheckResult, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:152:	results := make(map[string]HealthCheckResult, len(nodes))
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:159:			results[hostname] = HealthCheckResult{State: schema.MonitoringStateFailed}
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:169:			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "degraded metrics:", degradedList)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:172:			cclog.ComponentDebug("metricstore", "HealthCheck: node ", hostname, "missing metrics:", missingList)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/healthcheck.go:190:		results[hostname] = HealthCheckResult{
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore_test.go:596:func TestHealthCheck(t *testing.T) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore_test.go:738:			results, err := ms.HealthCheck(tt.cluster, tt.nodes, tt.expectedMetrics)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore_test.go:740:				t.Errorf("HealthCheck() error = %v", err)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore_test.go:746:				t.Errorf("HealthCheck() returned %d results, want %d", len(results), len(tt.nodes))
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore_test.go:753:					t.Errorf("HealthCheck() missing result for node %s", node)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore_test.go:760:						t.Errorf("HealthCheck() node %s status = %v, want %v", node, state.State, wantStatus)
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore_test.go:877:// TestBufferHealthChecks tests the buffer-level health check functions
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore_test.go:878:func TestBufferHealthChecks(t *testing.T) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/query.go:45:// HealthCheck delegates to the internal MemoryStore's HealthCheck.
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/query.go:46:func (ccms *InternalMetricStore) HealthCheck(cluster string,
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/query.go:48:) (map[string]HealthCheckResult, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/query.go:49:	return GetMemoryStore().HealthCheck(cluster, nodes, metrics)


<persisted-output>
Output too large (50.6KB). Full output saved to: /Users/jan/.claude/projects/-Users-jan-prg-ClusterCockpit-cc-backend/eec1fc87-d8f6-42f6-b296-d7a2f79af002/tool-results/toolu_01Tpv5NQGj3XaMoAoGqabqVR.txt

Preview (first 2KB):
     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements high-level query functions for loading job metric data
     7→// with automatic scope transformation and aggregation.
     8→//
     9→// Key Concepts:
    10→//
    11→// Metric Scopes: Metrics are collected at different granularities (native scope):
    12→//   - HWThread: Per hardware thread
    13→//   - Core: Per CPU core
    14→//   - Socket: Per CPU socket
    15→//   - MemoryDomain: Per memory domain (NUMA)
    16→//   - Accelerator: Per GPU/accelerator
    17→//   - Node: Per compute node
    18→//
    19→// Scope Transformation: The buildQueries functions transform between native scope
    20→// and requested scope by:
    21→//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
    22→//   - Rejecting requests for finer granularity than available
    23→//   - Handling special cases (e.g., Accelerator metrics)
    24→//
    25→// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
    26→// based on cluster topology and job resources.
    27→package metricstore
    28→
    29→import (
    30→	"context"
    31→	"fmt"
    32→	"strconv"
    33→	"strings"
    34→	"time"
    35→
    36→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    37→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    38→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    39→)
    40→
    41→type InternalMetricStore struct{}
    42→
    43→var MetricStoreHandle *InternalMetricStore
    44→
    45→// HealthCheck delegates to the internal MemoryStore's HealthCheck.
    46→func (ccms *InternalMetricStore) HealthCheck(cluster string,
    47→	nodes []string, metrics []string,
    48→) (map[string]HealthCheckResult, error) {
...
</persisted-output>


1357 /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/query.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements high-level query functions for loading job metric data
     7→// with automatic scope transformation and aggregation.
     8→//
     9→// Key Concepts:
    10→//
    11→// Metric Scopes: Metrics are collected at different granularities (native scope):
    12→//   - HWThread: Per hardware thread
    13→//   - Core: Per CPU core
    14→//   - Socket: Per CPU socket
    15→//   - MemoryDomain: Per memory domain (NUMA)
    16→//   - Accelerator: Per GPU/accelerator
    17→//   - Node: Per compute node
    18→//
    19→// Scope Transformation: The buildQueries functions transform between native scope
    20→// and requested scope by:
    21→//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
    22→//   - Rejecting requests for finer granularity than available
    23→//   - Handling special cases (e.g., Accelerator metrics)
    24→//
    25→// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
    26→// based on cluster topology and job resources.
    27→package metricstore
    28→
    29→import (
    30→	"context"
    31→	"fmt"
    32→	"strconv"
    33→	"strings"
    34→	"time"
    35→
    36→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    37→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    38→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    39→)
    40→
    41→type InternalMetricStore struct{}
    42→
    43→var MetricStoreHandle *InternalMetricStore
    44→
    45→// HealthCheck delegates to the internal MemoryStore's HealthCheck.
    46→func (ccms *InternalMetricStore) HealthCheck(cluster string,
    47→	nodes []string, metrics []string,
    48→) (map[string]HealthCheckResult, error) {
    49→	return GetMemoryStore().HealthCheck(cluster, nodes, metrics)
    50→}
    51→
    52→// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
    53→// When set to a non-nil function, LoadData will call this function instead of the default implementation.
    54→var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)
    55→
    56→// LoadData loads metric data for a specific job with automatic scope transformation.
    57→//
    58→// This is the primary function for retrieving job metric data. It handles:
    59→//   - Building queries with scope transformation via buildQueries
    60→//   - Fetching data from the metric store
    61→//   - Organizing results by metric and scope
    62→//   - Converting NaN statistics to 0 for JSON compatibility
    63→//   - Partial error handling (returns data for successful queries even if some fail)
    64→//
    65→// Parameters:
    66→//   - job: Job metadata including cluster, resources, and time range
    67→//   - metrics: List of metric names to load
    68→//   - scopes: Requested metric scopes (will be transformed to match native scopes)
    69→//   - ctx: Context for cancellation (currently unused but reserved for future use)
    70→//   - resolution: Data resolution in seconds (0 for native resolution)
    71→//
    72→// Returns:
    73→//   - JobData: Map of metric → scope → JobMetric with time-series data and statistics
    74→//   - Error: Returns error if query building or fetching fails, or partial error listing failed hosts
    75→//
    76→// Example:
    77→//
    78→//	jobData, err := LoadData(job, []string{"cpu_load", "mem_used"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 60)
    79→func (ccms *InternalMetricStore) LoadData(
    80→	job *schema.Job,
    81→	metrics []string,
    82→	scopes []schema.MetricScope,
    83→	ctx context.Context,
    84→	resolution int,
    85→) (schema.JobData, error) {
    86→	if TestLoadDataCallback != nil {
    87→		return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
    88→	}
    89→
    90→	queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
    91→	if err != nil {
    92→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
    93→		return nil, err
    94→	}
    95→
    96→	// Verify assignment is correct - log any inconsistencies for debugging
    97→	if len(queries) != len(assignedScope) {
    98→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
    99→			len(queries), len(assignedScope))
   100→	}
   101→
   102→	req := APIQueryRequest{
   103→		Cluster:   job.Cluster,
   104→		From:      job.StartTime,
   105→		To:        job.StartTime + int64(job.Duration),
   106→		Queries:   queries,
   107→		WithStats: true,
   108→		WithData:  true,
   109→	}
   110→
   111→	resBody, err := FetchData(req)
   112→	if err != nil {
   113→		cclog.Errorf("Error while fetching data : %s", err.Error())
   114→		return nil, err
   115→	}
   116→
   117→	var errors []string
   118→	jobData := make(schema.JobData)
   119→
   120→	// Add safety check for potential index out of range errors
   121→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   122→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   123→			len(req.Queries), len(resBody.Results), len(assignedScope))
   124→		if len(resBody.Results) > len(req.Queries) {
   125→			resBody.Results = resBody.Results[:len(req.Queries)]
   126→		}
   127→		if len(assignedScope) > len(req.Queries) {
   128→			assignedScope = assignedScope[:len(req.Queries)]
   129→		}
   130→	}
   131→
   132→	for i, row := range resBody.Results {
   133→		// Safety check to prevent index out of range errors
   134→		if i >= len(req.Queries) || i >= len(assignedScope) {
   135→			cclog.Warnf("Index out of range prevented: i=%d, queries=%d, assignedScope=%d",
   136→				i, len(req.Queries), len(assignedScope))
   137→			continue
   138→		}
   139→
   140→		query := req.Queries[i]
   141→		metric := query.Metric
   142→		scope := assignedScope[i]
   143→		mc := archive.GetMetricConfig(job.Cluster, metric)
   144→
   145→		if mc == nil {
   146→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   147→			continue
   148→		}
   149→
   150→		if _, ok := jobData[metric]; !ok {


/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store-queries.go
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go


Found 4 files
pkg/metricstore/query.go
internal/metricdispatch/metricdata.go
cmd/cc-backend/main.go
internal/api/api_test.go


grep -rn "MetricStoreHandle\s*=" /Users/jan/prg/ClusterCockpit/cc-backend --include="*.go" | head -20


/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go:329:		metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go:332:		metricstore.MetricStoreHandle = nil
/Users/jan/prg/ClusterCockpit/cc-backend/internal/api/api_test.go:153:	metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricdispatch/metricdata.go:106:				if metricstore.MetricStoreHandle == nil {


   320→	mscfg := ccconf.GetPackageConfig("metric-store")
   321→	if mscfg != nil {
   322→		metrics := metricstore.BuildMetricList()
   323→		metricstore.Init(mscfg, metrics, &wg)
   324→
   325→		// Inject repository as NodeProvider to break import cycle
   326→		ms := metricstore.GetMemoryStore()
   327→		jobRepo := repository.GetJobRepository()
   328→		ms.SetNodeProvider(jobRepo)
   329→		metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
   330→		haveMetricstore = true
   331→	} else {
   332→		metricstore.MetricStoreHandle = nil
   333→		cclog.Debug("missing internal metricstore configuration")
   334→	}
   335→
   336→	// Initialize external metric stores if configuration is provided
   337→	mscfg = ccconf.GetPackageConfig("metric-store-external")
   338→	if mscfg != nil {
   339→		err := metricdispatch.Init(mscfg)
   340→
   341→		if err != nil {
   342→			cclog.Debugf("error while initializing external metricdispatch: %v", err)
   343→		} else {
   344→			haveMetricstore = true
   345→		}
   346→	}
   347→
   348→	if !haveMetricstore {
   349→		return fmt.Errorf("missing metricstore configuration")
   350→	}
   351→
   352→	// Start archiver and task manager
   353→	archiver.Start(repository.GetJobRepository(), ctx)
   354→	taskmanager.Start(ccconf.GetPackageConfig("cron"), ccconf.GetPackageConfig("archive"))
   355→
   356→	// Initialize web UI
   357→	cfg := ccconf.GetPackageConfig("ui")
   358→	if err := web.Init(cfg); err != nil {
   359→		return fmt.Errorf("initializing web UI: %w", err)
   360→	}
   361→
   362→	// Initialize HTTP server
   363→	srv, err := NewServer(version, commit, date)
   364→	if err != nil {
   365→		return fmt.Errorf("creating server: %w", err)
   366→	}
   367→
   368→	// Channel to collect errors from server
   369→	errChan := make(chan error, 1)


Found 2 files
internal/metricstoreclient/cc-metric-store.go
pkg/metricstore/api.go


grep -A 5 "type APIQuery struct" /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go /Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go:type APIQuery struct {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-	Type        *string      `json:"type,omitempty"`
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-	SubType     *string      `json:"subtype,omitempty"`
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-	Metric      string       `json:"metric"`
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-	Hostname    string       `json:"host"`
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/api.go-	Resolution  int64        `json:"resolution"`
--
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go:type APIQuery struct {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-	Type       *string  `json:"type,omitempty"`        // Scope type (e.g., "core", "socket")
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-	SubType    *string  `json:"subtype,omitempty"`     // Sub-scope type (reserved for future use)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-	Metric     string   `json:"metric"`                // Metric name
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-	Hostname   string   `json:"host"`                  // Target hostname
/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go-	Resolution int      `json:"resolution"`            // Data resolution in seconds (0 = native)


Perfect! Now I have all the information needed. Let me provide a comprehensive report.

## Summary: Shared Interface & Implementation Pattern

I've found the complete picture of how the two metric store implementations work together. Here are the key findings:

### 1. **Shared Interface: `MetricDataRepository`**

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricdispatch/metricdata.go` (lines 21-60)

This is the central interface that both implementations must satisfy:

```go
type MetricDataRepository interface {
	// Load job metric data with specified metrics and scopes
	LoadData(job *schema.Job,
		metrics []string,
		scopes []schema.MetricScope,
		ctx context.Context,
		resolution int) (schema.JobData, error)

	// Load node-scope statistics only (faster than LoadData)
	LoadStats(job *schema.Job,
		metrics []string,
		ctx context.Context) (map[string]map[string]schema.MetricStatistics, error)

	// Load statistics across multiple scopes
	LoadScopedStats(job *schema.Job,
		metrics []string,
		scopes []schema.MetricScope,
		ctx context.Context) (schema.ScopedJobStats, error)

	// Load current metric data for specified nodes (systems view)
	LoadNodeData(cluster string,
		metrics, nodes []string,
		scopes []schema.MetricScope,
		from, to time.Time,
		ctx context.Context) (map[string]map[string][]*schema.JobMetric, error)

	// Load paginated node list data with scope organization
	LoadNodeListData(cluster, subCluster string,
		nodes []string,
		metrics []string,
		scopes []schema.MetricScope,
		resolution int,
		from, to time.Time,
		ctx context.Context) (map[string]schema.JobData, error)

	// Health check for cluster monitoring state
	HealthCheck(cluster string,
		nodes []string,
		metrics []string) (map[string]metricstore.HealthCheckResult, error)
}
```

### 2. **Implementation 1: Internal Metric Store**

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/query.go` (lines 41-50)

```go
type InternalMetricStore struct{}

var MetricStoreHandle *InternalMetricStore

// Implements all MetricDataRepository methods
func (ccms *InternalMetricStore) LoadData(...) (schema.JobData, error)
func (ccms *InternalMetricStore) LoadStats(...) (map[string]map[string]schema.MetricStatistics, error)
// ... and others
```

**Key file:** `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/query.go` contains all method implementations for `InternalMetricStore`

### 3. **Implementation 2: External CC-Metric-Store Client**

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go` (lines 71-79)

```go
type CCMetricStore struct {
	client        http.Client
	jwt           string
	url           string
	queryEndpoint string
	topologyCache map[string]*schema.Topology
}

// Implements all MetricDataRepository methods
func (ccms *CCMetricStore) LoadData(...) (schema.JobData, error)
func (ccms *CCMetricStore) LoadStats(...) (map[string]map[string]schema.MetricStatistics, error)
// ... and others
```

### 4. **Shared Types - `APIQuery` & `APIMetricData`**

Both packages define their **own copies** of these types (NOT shared):

**In `pkg/metricstore/api.go` (lines 74-93):**
```go
type APIQuery struct {
	Type       *string      `json:"type,omitempty"`
	SubType    *string      `json:"subtype,omitempty"`
	Metric     string       `json:"metric"`
	Hostname   string       `json:"host"`
	Resolution int64        `json:"resolution"`
	TypeIds    []string     `json:"type-ids,omitempty"`
	SubTypeIds []string     `json:"subtype-ids,omitempty"`
	ScaleFactor schema.Float `json:"scale-by,omitempty"`
	Aggregate  bool         `json:"aggreg"`
}
```

**In `internal/metricstoreclient/cc-metric-store.go` (lines 93-104):**
```go
type APIQuery struct {
	Type       *string  `json:"type,omitempty"`
	SubType    *string  `json:"subtype,omitempty"`
	Metric     string   `json:"metric"`
	Hostname   string   `json:"host"`
	Resolution int      `json:"resolution"`
	TypeIds    []string `json:"type-ids,omitempty"`
	SubTypeIds []string `json:"subtype-ids,omitempty"`
	Aggregate  bool     `json:"aggreg"`
}
```

**Note:** These are separate definitions (not shared) - they're almost identical but used independently by each package.

### 5. **Registration/Initialization**

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go` (lines 320-346)

```go
// Initialize internal metric store
mscfg := ccconf.GetPackageConfig("metric-store")
if mscfg != nil {
	metricstore.Init(mscfg, metrics, &wg)
	ms := metricstore.GetMemoryStore()
	jobRepo := repository.GetJobRepository()
	ms.SetNodeProvider(jobRepo)
	metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}
	haveMetricstore = true
}

// Initialize external metric store clients
mscfg = ccconf.GetPackageConfig("metric-store-external")
if mscfg != nil {
	err := metricdispatch.Init(mscfg)
	// ...
}
```

### 6. **Runtime Selection via `metricdispatch`**

**Location:** `/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricdispatch/metricdata.go` (lines 92-117)

The `GetMetricDataRepo()` function returns the appropriate implementation based on cluster/subcluster configuration:

```go
var metricDataRepos map[string]MetricDataRepository = map[string]MetricDataRepository{}

func GetMetricDataRepo(cluster string, subcluster string) (MetricDataRepository, error) {
	// Fallback chain:
	// 1. Try cluster + subcluster config
	// 2. Try cluster config only
	// 3. Try wildcard config
	// 4. Fall back to internal metric store (if available)
	// 5. Return error if none found

	if !ok && metricstore.MetricStoreHandle == nil {
		return nil, fmt.Errorf("[METRICDISPATCH]> no metric data repository configured")
	}
	repo = metricstore.MetricStoreHandle // Use internal as fallback
}
```

### Key Architecture Points

1. **Single Interface, Multiple Implementations:** The `MetricDataRepository` interface in `metricdispatch` is the bridge
2. **Independent Type Definitions:** Each package defines its own `APIQuery` type for internal use
3. **Dispatcher Pattern:** `internal/metricdispatch` acts as a registry and selector
4. **Fallback to Internal:** If no external metric store is configured, falls back to `MetricStoreHandle`
5. **No Shared Types Package:** There's no common `types.go` file - each package is self-contained and defines what it needs

**Files involved:**
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricdispatch/metricdata.go` — Interface definition and registry
- `/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/query.go` — Internal implementation
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/metricstoreclient/cc-metric-store.go` — External client implementation
- `/Users/jan/prg/ClusterCockpit/cc-backend/cmd/cc-backend/main.go` — Initialization

agentId: ae13c6665fcb142fa (for resuming to continue this agent's work if needed)
<usage>total_tokens: 46619
tool_uses: 21
duration_ms: 38792</usage>

Prompt:
Find the MetricDataRepository interface or similar interface that both pkg/metricstore and internal/metricstoreclient implement. Search for:
1. Any shared interface definition (likely in pkg/ or internal/)
2. How the two metric store implementations are registered/used
3. Any shared types between the two packages (APIQuery, etc.)
4. Check if there's a common types file or if each package defines its own

Search in pkg/metricstore/, internal/metricstoreclient/, and any relevant interface definition files. Also check if there's a metricdata or similar package.