     1→module github.com/ClusterCockpit/cc-backend
     2→
     3→go 1.25.0
     4→
     5→tool (
     6→	github.com/99designs/gqlgen
     7→	github.com/swaggo/swag/cmd/swag
     8→)
     9→
    10→require (
    11→	github.com/99designs/gqlgen v0.17.88
    12→	github.com/ClusterCockpit/cc-lib/v2 v2.9.1
    13→	github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0
    14→	github.com/Masterminds/squirrel v1.5.4
    15→	github.com/aws/aws-sdk-go-v2 v1.41.3
    16→	github.com/aws/aws-sdk-go-v2/config v1.32.11
    17→	github.com/aws/aws-sdk-go-v2/credentials v1.19.11
    18→	github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0
    19→	github.com/coreos/go-oidc/v3 v3.17.0
    20→	github.com/expr-lang/expr v1.17.8
    21→	github.com/go-chi/chi/v5 v5.2.5
    22→	github.com/go-chi/cors v1.2.2
    23→	github.com/go-co-op/gocron/v2 v2.19.1
    24→	github.com/go-ldap/ldap/v3 v3.4.12
    25→	github.com/golang-jwt/jwt/v5 v5.3.1
    26→	github.com/golang-migrate/migrate/v4 v4.19.1
    27→	github.com/google/gops v0.3.29
    28→	github.com/gorilla/sessions v1.4.0
    29→	github.com/jmoiron/sqlx v1.4.0
    30→	github.com/joho/godotenv v1.5.1
    31→	github.com/mattn/go-sqlite3 v1.14.34
    32→	github.com/parquet-go/parquet-go v0.29.0
    33→	github.com/qustavo/sqlhooks/v2 v2.1.0
    34→	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
    35→	github.com/stretchr/testify v1.11.1
    36→	github.com/swaggo/http-swagger v1.3.4
    37→	github.com/swaggo/swag v1.16.6
    38→	github.com/vektah/gqlparser/v2 v2.5.32
    39→	golang.org/x/crypto v0.49.0
    40→	golang.org/x/oauth2 v0.36.0
    41→	golang.org/x/sync v0.20.0
    42→	golang.org/x/time v0.15.0
    43→)
    44→
    45→require (
    46→	github.com/Azure/go-ntlmssp v0.1.0 // indirect
    47→	github.com/KyleBanks/depth v1.2.1 // indirect
    48→	github.com/agnivade/levenshtein v1.2.1 // indirect
    49→	github.com/andybalholm/brotli v1.2.0 // indirect
    50→	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
    51→	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect
    52→	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect
    53→	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect
    54→	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect
    55→	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect
    56→	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect
    57→	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect
    58→	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect
    59→	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect
    60→	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect
    61→	github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect
    62→	github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect
    63→	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect
    64→	github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect
    65→	github.com/aws/smithy-go v1.24.2 // indirect
    66→	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
    67→	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
    68→	github.com/fsnotify/fsnotify v1.9.0 // indirect
    69→	github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
    70→	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
    71→	github.com/go-openapi/jsonpointer v0.22.5 // indirect
    72→	github.com/go-openapi/jsonreference v0.21.5 // indirect
    73→	github.com/go-openapi/spec v0.22.4 // indirect
    74→	github.com/go-openapi/swag/conv v0.25.5 // indirect
    75→	github.com/go-openapi/swag/jsonname v0.25.5 // indirect
    76→	github.com/go-openapi/swag/jsonutils v0.25.5 // indirect
    77→	github.com/go-openapi/swag/loading v0.25.5 // indirect
    78→	github.com/go-openapi/swag/stringutils v0.25.5 // indirect
    79→	github.com/go-openapi/swag/typeutils v0.25.5 // indirect
    80→	github.com/go-openapi/swag/yamlutils v0.25.5 // indirect
    81→	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
    82→	github.com/goccy/go-yaml v1.19.2 // indirect
    83→	github.com/google/uuid v1.6.0 // indirect
    84→	github.com/gorilla/securecookie v1.1.2 // indirect
    85→	github.com/gorilla/websocket v1.5.3 // indirect
    86→	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
    87→	github.com/influxdata/influxdb-client-go/v2 v2.14.0 // indirect
    88→	github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
    89→	github.com/jonboulle/clockwork v0.5.0 // indirect
    90→	github.com/klauspost/compress v1.18.4 // indirect
    91→	github.com/kr/pretty v0.3.1 // indirect
    92→	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
    93→	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
    94→	github.com/nats-io/nats.go v1.49.0 // indirect
    95→	github.com/nats-io/nkeys v0.4.15 // indirect
    96→	github.com/nats-io/nuid v1.0.1 // indirect
    97→	github.com/oapi-codegen/runtime v1.2.0 // indirect
    98→	github.com/parquet-go/bitpack v1.0.0 // indirect
    99→	github.com/parquet-go/jsonlite v1.4.0 // indirect
   100→	github.com/pierrec/lz4/v4 v4.1.26 // indirect
   101→	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
   102→	github.com/robfig/cron/v3 v3.0.1 // indirect
   103→	github.com/rogpeppe/go-internal v1.10.0 // indirect
   104→	github.com/russross/blackfriday/v2 v2.1.0 // indirect
   105→	github.com/sosodev/duration v1.4.0 // indirect
   106→	github.com/stmcginnis/gofish v0.21.4 // indirect
   107→	github.com/stretchr/objx v0.5.2 // indirect
   108→	github.com/swaggo/files v1.0.1 // indirect
   109→	github.com/twpayne/go-geom v1.6.1 // indirect
   110→	github.com/urfave/cli/v2 v2.27.7 // indirect
   111→	github.com/urfave/cli/v3 v3.7.0 // indirect
   112→	github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
   113→	go.yaml.in/yaml/v2 v2.4.3 // indirect
   114→	go.yaml.in/yaml/v3 v3.0.4 // indirect
   115→	golang.org/x/mod v0.34.0 // indirect
   116→	golang.org/x/net v0.52.0 // indirect
   117→	golang.org/x/sys v0.42.0 // indirect
   118→	golang.org/x/text v0.35.0 // indirect
   119→	golang.org/x/tools v0.43.0 // indirect
   120→	google.golang.org/protobuf v1.36.11 // indirect
   121→	gopkg.in/yaml.v3 v3.0.1 // indirect
   122→	sigs.k8s.io/yaml v1.6.0 // indirect
   123→)
   124→

module github.com/ClusterCockpit/cc-backend

go 1.25.0

tool (
	github.com/99designs/gqlgen
	github.com/swaggo/swag/cmd/swag
)

require (
	github.com/99designs/gqlgen v0.17.88
	github.com/ClusterCockpit/cc-lib/v2 v2.9.1
	github.com/ClusterCockpit/cc-line-protocol/v2 v2.4.0
	github.com/Masterminds/squirrel v1.5.4
	github.com/aws/aws-sdk-go-v2 v1.41.3
	github.com/aws/aws-sdk-go-v2/config v1.32.11
	github.com/aws/aws-sdk-go-v2/credentials v1.19.11
	github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0
	github.com/coreos/go-oidc/v3 v3.17.0
	github.com/expr-lang/expr v1.17.8
	github.com/go-chi/chi/v5 v5.2.5
	github.com/go-chi/cors v1.2.2
	github.com/go-co-op/gocron/v2 v2.19.1
	github.com/go-ldap/ldap/v3 v3.4.12
	github.com/golang-jwt/jwt/v5 v5.3.1
	github.com/golang-migrate/migrate/v4 v4.19.1
	github.com/google/gops v0.3.29
	github.com/gorilla/sessions v1.4.0
	github.com/jmoiron/sqlx v1.4.0
	github.com/joho/godotenv v1.5.1
	github.com/mattn/go-sqlite3 v1.14.34
	github.com/parquet-go/parquet-go v0.29.0
	github.com/qustavo/sqlhooks/v2 v2.1.0
	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
	github.com/stretchr/testify v1.11.1
	github.com/swaggo/http-swagger v1.3.4
	github.com/swaggo/swag v1.16.6
	github.com/vektah/gqlparser/v2 v2.5.32
	golang.org/x/crypto v0.49.0
	golang.org/x/oauth2 v0.36.0
	golang.org/x/sync v0.20.0
	golang.org/x/time v0.15.0
)

require (
	github.com/Azure/go-ntlmssp v0.1.0 // indirect
	github.com/KyleBanks/depth v1.2.1 // indirect
	github.com/agnivade/levenshtein v1.2.1 // indirect
	github.com/andybalholm/brotli v1.2.0 // indirect
	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect
	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect
	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect
	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect
	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect
	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect
	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect
	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect
	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect
	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect
	github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect
	github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect
	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect
	github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect
	github.com/aws/smithy-go v1.24.2 // indirect
	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
	github.com/fsnotify/fsnotify v1.9.0 // indirect
	github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
	github.com/go-openapi/jsonpointer v0.22.5 // indirect
	github.com/go-openapi/jsonreference v0.21.5 // indirect
	github.com/go-openapi/spec v0.22.4 // indirect
	github.com/go-openapi/swag/conv v0.25.5 // indirect
	github.com/go-openapi/swag/jsonname v0.25.5 // indirect
	github.com/go-openapi/swag/jsonutils v0.25.5 // indirect
	github.com/go-openapi/swag/loading v0.25.5 // indirect
	github.com/go-openapi/swag/stringutils v0.25.5 // indirect
	github.com/go-openapi/swag/typeutils v0.25.5 // indirect
	github.com/go-openapi/swag/yamlutils v0.25.5 // indirect
	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
	github.com/goccy/go-yaml v1.19.2 // indirect
	github.com/google/uuid v1.6.0 // indirect
	github.com/gorilla/securecookie v1.1.2 // indirect
	github.com/gorilla/websocket v1.5.3 // indirect
	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
	github.com/influxdata/influxdb-client-go/v2 v2.14.0 // indirect
	github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
	github.com/jonboulle/clockwork v0.5.0 // indirect
	github.com/klauspost/compress v1.18.4 // indirect
	github.com/kr/pretty v0.3.1 // indirect
	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
	github.com/nats-io/nats.go v1.49.0 // indirect
	github.com/nats-io/nkeys v0.4.15 // indirect
	github.com/nats-io/nuid v1.0.1 // indirect
	github.com/oapi-codegen/runtime v1.2.0 // indirect
	github.com/parquet-go/bitpack v1.0.0 // indirect
	github.com/parquet-go/jsonlite v1.4.0 // indirect
	github.com/pierrec/lz4/v4 v4.1.26 // indirect
	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
	github.com/robfig/cron/v3 v3.0.1 // indirect
	github.com/rogpeppe/go-internal v1.10.0 // indirect
	github.com/russross/blackfriday/v2 v2.1.0 // indirect
	github.com/sosodev/duration v1.4.0 // indirect
	github.com/stmcginnis/gofish v0.21.4 // indirect
	github.com/stretchr/objx v0.5.2 // indirect
	github.com/swaggo/files v1.0.1 // indirect
	github.com/twpayne/go-geom v1.6.1 // indirect
	github.com/urfave/cli/v2 v2.27.7 // indirect
	github.com/urfave/cli/v3 v3.7.0 // indirect
	github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
	go.yaml.in/yaml/v2 v2.4.3 // indirect
	go.yaml.in/yaml/v3 v3.0.4 // indirect
	golang.org/x/mod v0.34.0 // indirect
	golang.org/x/net v0.52.0 // indirect
	golang.org/x/sys v0.42.0 // indirect
	golang.org/x/text v0.35.0 // indirect
	golang.org/x/tools v0.43.0 // indirect
	google.golang.org/protobuf v1.36.11 // indirect
	gopkg.in/yaml.v3 v3.0.1 // indirect
	sigs.k8s.io/yaml v1.6.0 // indirect
)


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file implements high-level query functions for loading job metric data
     7→// with automatic scope transformation and aggregation.
     8→//
     9→// Key Concepts:
    10→//
    11→// Metric Scopes: Metrics are collected at different granularities (native scope):
    12→//   - HWThread: Per hardware thread
    13→//   - Core: Per CPU core
    14→//   - Socket: Per CPU socket
    15→//   - MemoryDomain: Per memory domain (NUMA)
    16→//   - Accelerator: Per GPU/accelerator
    17→//   - Node: Per compute node
    18→//
    19→// Scope Transformation: The buildQueries functions transform between native scope
    20→// and requested scope by:
    21→//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
    22→//   - Rejecting requests for finer granularity than available
    23→//   - Handling special cases (e.g., Accelerator metrics)
    24→//
    25→// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
    26→// based on cluster topology and job resources.
    27→package metricstore
    28→
    29→import (
    30→	"context"
    31→	"fmt"
    32→	"strings"
    33→	"time"
    34→
    35→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    36→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    37→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    38→)
    39→
    40→type InternalMetricStore struct{}
    41→
    42→var MetricStoreHandle *InternalMetricStore
    43→
    44→// HealthCheck delegates to the internal MemoryStore's HealthCheck.
    45→func (ccms *InternalMetricStore) HealthCheck(cluster string,
    46→	nodes []string, metrics []string,
    47→) (map[string]HealthCheckResult, error) {
    48→	return GetMemoryStore().HealthCheck(cluster, nodes, metrics)
    49→}
    50→
    51→// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
    52→// When set to a non-nil function, LoadData will call this function instead of the default implementation.
    53→var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)
    54→
    55→// LoadData loads metric data for a specific job with automatic scope transformation.
    56→//
    57→// This is the primary function for retrieving job metric data. It handles:
    58→//   - Building queries with scope transformation via buildQueries
    59→//   - Fetching data from the metric store
    60→//   - Organizing results by metric and scope
    61→//   - Converting NaN statistics to 0 for JSON compatibility
    62→//   - Partial error handling (returns data for successful queries even if some fail)
    63→//
    64→// Parameters:
    65→//   - job: Job metadata including cluster, resources, and time range
    66→//   - metrics: List of metric names to load
    67→//   - scopes: Requested metric scopes (will be transformed to match native scopes)
    68→//   - ctx: Context for cancellation (currently unused but reserved for future use)
    69→//   - resolution: Data resolution in seconds (0 for native resolution)
    70→//
    71→// Returns:
    72→//   - JobData: Map of metric → scope → JobMetric with time-series data and statistics
    73→//   - Error: Returns error if query building or fetching fails, or partial error listing failed hosts
    74→//
    75→// Example:
    76→//
    77→//	jobData, err := LoadData(job, []string{"cpu_load", "mem_used"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 60)
    78→func (ccms *InternalMetricStore) LoadData(
    79→	job *schema.Job,
    80→	metrics []string,
    81→	scopes []schema.MetricScope,
    82→	ctx context.Context,
    83→	resolution int,
    84→) (schema.JobData, error) {
    85→	if TestLoadDataCallback != nil {
    86→		return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
    87→	}
    88→
    89→	queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
    90→	if err != nil {
    91→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
    92→		return nil, err
    93→	}
    94→
    95→	// Verify assignment is correct - log any inconsistencies for debugging
    96→	if len(queries) != len(assignedScope) {
    97→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
    98→			len(queries), len(assignedScope))
    99→	}
   100→
   101→	req := APIQueryRequest{
   102→		Cluster:   job.Cluster,
   103→		From:      job.StartTime,
   104→		To:        job.StartTime + int64(job.Duration),
   105→		Queries:   queries,
   106→		WithStats: true,
   107→		WithData:  true,
   108→	}
   109→
   110→	resBody, err := FetchData(req)
   111→	if err != nil {
   112→		cclog.Errorf("Error while fetching data : %s", err.Error())
   113→		return nil, err
   114→	}
   115→
   116→	var errors []string
   117→	jobData := make(schema.JobData)
   118→
   119→	// Add safety check for potential index out of range errors
   120→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   121→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   122→			len(req.Queries), len(resBody.Results), len(assignedScope))
   123→		if len(resBody.Results) > len(req.Queries) {
   124→			resBody.Results = resBody.Results[:len(req.Queries)]
   125→		}
   126→		if len(assignedScope) > len(req.Queries) {
   127→			assignedScope = assignedScope[:len(req.Queries)]
   128→		}
   129→	}
   130→
   131→	for i, row := range resBody.Results {
   132→		query := req.Queries[i]
   133→		metric := query.Metric
   134→		scope := assignedScope[i]
   135→		mc := archive.GetMetricConfig(job.Cluster, metric)
   136→
   137→		if mc == nil {
   138→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
   139→			continue
   140→		}
   141→
   142→		if _, ok := jobData[metric]; !ok {
   143→			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
   144→		}
   145→
   146→		res := mc.Timestep
   147→		if len(row) > 0 {
   148→			res = int(row[0].Resolution)
   149→		}
   150→
   151→		jobMetric, ok := jobData[metric][scope]
   152→		if !ok {
   153→			jobMetric = &schema.JobMetric{
   154→				Unit:     mc.Unit,
   155→				Timestep: res,
   156→				Series:   make([]schema.Series, 0),
   157→			}
   158→			jobData[metric][scope] = jobMetric
   159→		}
   160→
   161→		for ndx, res := range row {
   162→			if res.Error != nil {
   163→				/* Build list for "partial errors", if any */
   164→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   165→				continue
   166→			}
   167→
   168→			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)
   169→
   170→			SanitizeStats(&res.Avg, &res.Min, &res.Max)
   171→
   172→			jobMetric.Series = append(jobMetric.Series, schema.Series{
   173→				Hostname: query.Hostname,
   174→				ID:       id,
   175→				Statistics: schema.MetricStatistics{
   176→					Avg: float64(res.Avg),
   177→					Min: float64(res.Min),
   178→					Max: float64(res.Max),
   179→				},
   180→				Data: res.Data,
   181→			})
   182→		}
   183→
   184→		// So that one can later check len(jobData):
   185→		if len(jobMetric.Series) == 0 {
   186→			delete(jobData[metric], scope)
   187→			if len(jobData[metric]) == 0 {
   188→				delete(jobData, metric)
   189→			}
   190→		}
   191→	}
   192→
   193→	if len(errors) != 0 {
   194→		/* Returns list for "partial errors" */
   195→		return jobData, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   196→	}
   197→	return jobData, nil
   198→}
   199→
   200→// buildQueries constructs APIQuery structures with automatic scope transformation for a job.
   201→//
   202→// This function implements the core scope transformation logic, handling all combinations of
   203→// native metric scopes and requested scopes. It uses the cluster topology to determine which
   204→// hardware IDs to include in each query.
   205→//
   206→// Scope Transformation Rules:
   207→//   - If native scope >= requested scope: Aggregates data (Aggregate=true in APIQuery)
   208→//   - If native scope < requested scope: Returns error (cannot increase granularity)
   209→//   - Special handling for Accelerator scope (independent of CPU hierarchy)
   210→//
   211→// The function generates one or more APIQuery per (metric, scope, host) combination:
   212→//   - For non-aggregated queries: One query with all relevant IDs
   213→//   - For aggregated queries: May generate multiple queries (e.g., one per socket/core)
   214→//
   215→// Parameters:
   216→//   - job: Job metadata including cluster, subcluster, and resource allocation
   217→//   - metrics: List of metrics to query
   218→//   - scopes: Requested scopes for each metric
   219→//   - resolution: Data resolution in seconds
   220→//
   221→// Returns:
   222→//   - []APIQuery: List of queries to execute
   223→//   - []schema.MetricScope: Assigned scope for each query (after transformation)
   224→//   - error: Returns error if topology lookup fails or unhandled scope combination encountered
   225→func buildQueries(
   226→	job *schema.Job,
   227→	metrics []string,
   228→	scopes []schema.MetricScope,
   229→	resolution int64,
   230→) ([]APIQuery, []schema.MetricScope, error) {
   231→	if len(job.Resources) == 0 {
   232→		return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > no resources allocated for job %d", job.JobID)
   233→	}
   234→
   235→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
   236→	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(job.Resources))
   237→
   238→	subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster)
   239→	if scerr != nil {
   240→		return nil, nil, scerr
   241→	}
   242→	topology := subcluster.Topology
   243→
   244→	for _, metric := range metrics {
   245→		mc := archive.GetMetricConfig(job.Cluster, metric)
   246→		if mc == nil {
   247→			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
   248→			continue
   249→		}
   250→
   251→		// Skip if metric is removed for subcluster
   252→		if len(mc.SubClusters) != 0 && IsMetricRemovedForSubCluster(mc, job.SubCluster) {
   253→			continue
   254→		}
   255→
   256→		// Avoid duplicates...
   257→		handledScopes := make([]schema.MetricScope, 0, 3)
   258→
   259→	scopesLoop:
   260→		for _, requestedScope := range scopes {
   261→			nativeScope := mc.Scope
   262→			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
   263→				continue
   264→			}
   265→
   266→			scope := nativeScope.Max(requestedScope)
   267→			for _, s := range handledScopes {
   268→				if scope == s {
   269→					continue scopesLoop
   270→				}
   271→			}
   272→			handledScopes = append(handledScopes, scope)
   273→
   274→			for _, host := range job.Resources {
   275→				hwthreads := host.HWThreads
   276→				if hwthreads == nil {
   277→					hwthreads = topology.Node
   278→				}
   279→
   280→				scopeResults, ok := BuildScopeQueries(
   281→					nativeScope, requestedScope,
   282→					metric, host.Hostname,
   283→					&topology, hwthreads, host.Accelerators,
   284→				)
   285→
   286→				if !ok {
   287→					return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   288→				}
   289→
   290→				for _, sr := range scopeResults {
   291→					queries = append(queries, APIQuery{
   292→						Metric:     sr.Metric,
   293→						Hostname:   sr.Hostname,
   294→						Aggregate:  sr.Aggregate,
   295→						Type:       sr.Type,
   296→						TypeIds:    sr.TypeIds,
   297→						Resolution: resolution,
   298→					})
   299→					assignedScope = append(assignedScope, sr.Scope)
   300→				}
   301→			}
   302→		}
   303→	}
   304→
   305→	return queries, assignedScope, nil
   306→}
   307→
   308→// LoadStats loads only metric statistics (avg/min/max) for a job at node scope.
   309→//
   310→// This is an optimized version of LoadData that fetches only statistics without
   311→// time-series data, reducing bandwidth and memory usage. Always queries at node scope.
   312→//
   313→// Parameters:
   314→//   - job: Job metadata
   315→//   - metrics: List of metric names
   316→//   - ctx: Context (currently unused)
   317→//
   318→// Returns:
   319→//   - Map of metric → hostname → statistics
   320→//   - Error on query building or fetching failure
   321→func (ccms *InternalMetricStore) LoadStats(
   322→	job *schema.Job,
   323→	metrics []string,
   324→	ctx context.Context,
   325→) (map[string]map[string]schema.MetricStatistics, error) {
   326→	// TODO(#166): Add scope parameter for analysis view accelerator normalization
   327→	queries, _, err := buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0)
   328→	if err != nil {
   329→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
   330→		return nil, err
   331→	}
   332→
   333→	req := APIQueryRequest{
   334→		Cluster:   job.Cluster,
   335→		From:      job.StartTime,
   336→		To:        job.StartTime + int64(job.Duration),
   337→		Queries:   queries,
   338→		WithStats: true,
   339→		WithData:  false,
   340→	}
   341→
   342→	resBody, err := FetchData(req)
   343→	if err != nil {
   344→		cclog.Errorf("Error while fetching data : %s", err.Error())
   345→		return nil, err
   346→	}
   347→
   348→	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
   349→	for i, res := range resBody.Results {
   350→		if i >= len(req.Queries) {
   351→			cclog.Warnf("LoadStats: result index %d exceeds queries length %d", i, len(req.Queries))
   352→			break
   353→		}
   354→		if len(res) == 0 {
   355→			// No Data Found For Metric, Logged in FetchData to Warn
   356→			continue
   357→		}
   358→		query := req.Queries[i]
   359→		metric := query.Metric
   360→		data := res[0]
   361→		if data.Error != nil {
   362→			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
   363→			continue
   364→		}
   365→
   366→		metricdata, ok := stats[metric]
   367→		if !ok {
   368→			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
   369→			stats[metric] = metricdata
   370→		}
   371→
   372→		if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() {
   373→			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
   374→			continue
   375→		}
   376→
   377→		metricdata[query.Hostname] = schema.MetricStatistics{
   378→			Avg: float64(data.Avg),
   379→			Min: float64(data.Min),
   380→			Max: float64(data.Max),
   381→		}
   382→	}
   383→
   384→	return stats, nil
   385→}
   386→
   387→// LoadScopedStats loads metric statistics for a job with scope-aware grouping.
   388→//
   389→// Similar to LoadStats but supports multiple scopes and returns statistics grouped
   390→// by scope with hardware IDs (e.g., per-core, per-socket statistics).
   391→//
   392→// Parameters:
   393→//   - job: Job metadata
   394→//   - metrics: List of metric names
   395→//   - scopes: Requested metric scopes
   396→//   - ctx: Context (currently unused)
   397→//
   398→// Returns:
   399→//   - ScopedJobStats: Map of metric → scope → []ScopedStats (with hostname and ID)
   400→//   - Error or partial error listing failed queries
   401→func (ccms *InternalMetricStore) LoadScopedStats(
   402→	job *schema.Job,
   403→	metrics []string,
   404→	scopes []schema.MetricScope,
   405→	ctx context.Context,
   406→) (schema.ScopedJobStats, error) {
   407→	queries, assignedScope, err := buildQueries(job, metrics, scopes, 0)
   408→	if err != nil {
   409→		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
   410→		return nil, err
   411→	}
   412→
   413→	req := APIQueryRequest{
   414→		Cluster:   job.Cluster,
   415→		From:      job.StartTime,
   416→		To:        job.StartTime + int64(job.Duration),
   417→		Queries:   queries,
   418→		WithStats: true,
   419→		WithData:  false,
   420→	}
   421→
   422→	resBody, err := FetchData(req)
   423→	if err != nil {
   424→		cclog.Errorf("Error while fetching data : %s", err.Error())
   425→		return nil, err
   426→	}
   427→
   428→	var errors []string
   429→	scopedJobStats := make(schema.ScopedJobStats)
   430→
   431→	for i, row := range resBody.Results {
   432→		if len(row) == 0 {
   433→			// No Data Found For Metric, Logged in FetchData to Warn
   434→			continue
   435→		}
   436→		query := req.Queries[i]
   437→		metric := query.Metric
   438→		scope := assignedScope[i]
   439→
   440→		if _, ok := scopedJobStats[metric]; !ok {
   441→			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
   442→		}
   443→
   444→		if _, ok := scopedJobStats[metric][scope]; !ok {
   445→			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
   446→		}
   447→
   448→		for ndx, res := range row {
   449→			if res.Error != nil {
   450→				/* Build list for "partial errors", if any */
   451→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   452→				continue
   453→			}
   454→
   455→			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)
   456→
   457→			SanitizeStats(&res.Avg, &res.Min, &res.Max)
   458→
   459→			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
   460→				Hostname: query.Hostname,
   461→				ID:       id,
   462→				Data: &schema.MetricStatistics{
   463→					Avg: float64(res.Avg),
   464→					Min: float64(res.Min),
   465→					Max: float64(res.Max),
   466→				},
   467→			})
   468→		}
   469→
   470→		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
   471→		if len(scopedJobStats[metric][scope]) == 0 {
   472→			delete(scopedJobStats[metric], scope)
   473→			if len(scopedJobStats[metric]) == 0 {
   474→				delete(scopedJobStats, metric)
   475→			}
   476→		}
   477→	}
   478→
   479→	if len(errors) != 0 {
   480→		/* Returns list for "partial errors" */
   481→		return scopedJobStats, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   482→	}
   483→	return scopedJobStats, nil
   484→}
   485→
   486→// LoadNodeData loads metric data for specific nodes in a cluster over a time range.
   487→//
   488→// Unlike LoadData which operates on job resources, this function queries arbitrary nodes
   489→// directly. Useful for system monitoring and node status views.
   490→//
   491→// Parameters:
   492→//   - cluster: Cluster name
   493→//   - metrics: List of metric names
   494→//   - nodes: List of node hostnames (nil = all nodes in cluster via ForAllNodes)
   495→//   - scopes: Requested metric scopes (currently unused - always node scope)
   496→//   - from, to: Time range
   497→//   - ctx: Context (currently unused)
   498→//
   499→// Returns:
   500→//   - Map of hostname → metric → []JobMetric
   501→//   - Error or partial error listing failed queries
   502→func (ccms *InternalMetricStore) LoadNodeData(
   503→	cluster string,
   504→	metrics, nodes []string,
   505→	scopes []schema.MetricScope,
   506→	from, to time.Time,
   507→	ctx context.Context,
   508→) (map[string]map[string][]*schema.JobMetric, error) {
   509→	req := APIQueryRequest{
   510→		Cluster:   cluster,
   511→		From:      from.Unix(),
   512→		To:        to.Unix(),
   513→		WithStats: true,
   514→		WithData:  true,
   515→	}
   516→
   517→	if nodes == nil {
   518→		req.ForAllNodes = append(req.ForAllNodes, metrics...)
   519→	} else {
   520→		for _, node := range nodes {
   521→			for _, metric := range metrics {
   522→				req.Queries = append(req.Queries, APIQuery{
   523→					Hostname:   node,
   524→					Metric:     metric,
   525→					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
   526→				})
   527→			}
   528→		}
   529→	}
   530→
   531→	resBody, err := FetchData(req)
   532→	if err != nil {
   533→		cclog.Errorf("Error while fetching data : %s", err.Error())
   534→		return nil, err
   535→	}
   536→
   537→	var errors []string
   538→	data := make(map[string]map[string][]*schema.JobMetric)
   539→	for i, res := range resBody.Results {
   540→		if len(res) == 0 {
   541→			// No Data Found For Metric, Logged in FetchData to Warn
   542→			continue
   543→		}
   544→
   545→		var query APIQuery
   546→		if resBody.Queries != nil {
   547→			query = resBody.Queries[i]
   548→		} else {
   549→			query = req.Queries[i]
   550→		}
   551→
   552→		metric := query.Metric
   553→		qdata := res[0]
   554→		if qdata.Error != nil {
   555→			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
   556→			continue
   557→		}
   558→
   559→		mc := archive.GetMetricConfig(cluster, metric)
   560→		if mc == nil {
   561→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   562→			continue
   563→		}
   564→
   565→		SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)
   566→
   567→		hostdata, ok := data[query.Hostname]
   568→		if !ok {
   569→			hostdata = make(map[string][]*schema.JobMetric)
   570→			data[query.Hostname] = hostdata
   571→		}
   572→
   573→		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
   574→			Unit:     mc.Unit,
   575→			Timestep: mc.Timestep,
   576→			Series: []schema.Series{
   577→				{
   578→					Hostname: query.Hostname,
   579→					Data:     qdata.Data,
   580→					Statistics: schema.MetricStatistics{
   581→						Avg: float64(qdata.Avg),
   582→						Min: float64(qdata.Min),
   583→						Max: float64(qdata.Max),
   584→					},
   585→				},
   586→			},
   587→		})
   588→	}
   589→
   590→	if len(errors) != 0 {
   591→		/* Returns list of "partial errors" */
   592→		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   593→	}
   594→
   595→	return data, nil
   596→}
   597→
   598→// LoadNodeListData loads metric data for a list of nodes with full scope transformation support.
   599→//
   600→// This is the most flexible node data loading function, supporting arbitrary scopes and
   601→// resolution. Uses buildNodeQueries for proper scope transformation based on topology.
   602→//
   603→// Parameters:
   604→//   - cluster: Cluster name
   605→//   - subCluster: SubCluster name (empty string to infer from node names)
   606→//   - nodes: List of node hostnames
   607→//   - metrics: List of metric names
   608→//   - scopes: Requested metric scopes
   609→//   - resolution: Data resolution in seconds
   610→//   - from, to: Time range
   611→//   - ctx: Context (currently unused)
   612→//
   613→// Returns:
   614→//   - Map of hostname → JobData (metric → scope → JobMetric)
   615→//   - Error or partial error listing failed queries
   616→func (ccms *InternalMetricStore) LoadNodeListData(
   617→	cluster, subCluster string,
   618→	nodes []string,
   619→	metrics []string,
   620→	scopes []schema.MetricScope,
   621→	resolution int,
   622→	from, to time.Time,
   623→	ctx context.Context,
   624→) (map[string]schema.JobData, error) {
   625→	// Note: Order of node data is not guaranteed after this point
   626→	queries, assignedScope, err := buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution))
   627→	if err != nil {
   628→		cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
   629→		return nil, err
   630→	}
   631→
   632→	// Verify assignment is correct - log any inconsistencies for debugging
   633→	if len(queries) != len(assignedScope) {
   634→		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
   635→			len(queries), len(assignedScope))
   636→	}
   637→
   638→	req := APIQueryRequest{
   639→		Cluster:   cluster,
   640→		Queries:   queries,
   641→		From:      from.Unix(),
   642→		To:        to.Unix(),
   643→		WithStats: true,
   644→		WithData:  true,
   645→	}
   646→
   647→	resBody, err := FetchData(req)
   648→	if err != nil {
   649→		cclog.Errorf("Error while fetching data : %s", err.Error())
   650→		return nil, err
   651→	}
   652→
   653→	var errors []string
   654→	data := make(map[string]schema.JobData)
   655→
   656→	// Add safety check for index out of range issues
   657→	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
   658→		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
   659→			len(req.Queries), len(resBody.Results), len(assignedScope))
   660→		if len(resBody.Results) > len(req.Queries) {
   661→			resBody.Results = resBody.Results[:len(req.Queries)]
   662→		}
   663→		if len(assignedScope) > len(req.Queries) {
   664→			assignedScope = assignedScope[:len(req.Queries)]
   665→		}
   666→	}
   667→
   668→	for i, row := range resBody.Results {
   669→		var query APIQuery
   670→		if resBody.Queries != nil {
   671→			if i < len(resBody.Queries) {
   672→				query = resBody.Queries[i]
   673→			} else {
   674→				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
   675→					i, len(resBody.Queries))
   676→				continue
   677→			}
   678→		} else {
   679→			query = req.Queries[i]
   680→		}
   681→
   682→		metric := query.Metric
   683→		scope := assignedScope[i]
   684→		mc := archive.GetMetricConfig(cluster, metric)
   685→		if mc == nil {
   686→			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
   687→			continue
   688→		}
   689→
   690→		res := mc.Timestep
   691→		if len(row) > 0 {
   692→			res = int(row[0].Resolution)
   693→		}
   694→
   695→		// Init Nested Map Data Structures If Not Found
   696→		hostData, ok := data[query.Hostname]
   697→		if !ok {
   698→			hostData = make(schema.JobData)
   699→			data[query.Hostname] = hostData
   700→		}
   701→
   702→		metricData, ok := hostData[metric]
   703→		if !ok {
   704→			metricData = make(map[schema.MetricScope]*schema.JobMetric)
   705→			data[query.Hostname][metric] = metricData
   706→		}
   707→
   708→		scopeData, ok := metricData[scope]
   709→		if !ok {
   710→			scopeData = &schema.JobMetric{
   711→				Unit:     mc.Unit,
   712→				Timestep: res,
   713→				Series:   make([]schema.Series, 0),
   714→			}
   715→			data[query.Hostname][metric][scope] = scopeData
   716→		}
   717→
   718→		for ndx, res := range row {
   719→			if res.Error != nil {
   720→				/* Build list for "partial errors", if any */
   721→				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
   722→				continue
   723→			}
   724→
   725→			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)
   726→
   727→			SanitizeStats(&res.Avg, &res.Min, &res.Max)
   728→
   729→			scopeData.Series = append(scopeData.Series, schema.Series{
   730→				Hostname: query.Hostname,
   731→				ID:       id,
   732→				Statistics: schema.MetricStatistics{
   733→					Avg: float64(res.Avg),
   734→					Min: float64(res.Min),
   735→					Max: float64(res.Max),
   736→				},
   737→				Data: res.Data,
   738→			})
   739→		}
   740→	}
   741→
   742→	if len(errors) != 0 {
   743→		/* Returns list of "partial errors" */
   744→		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
   745→	}
   746→
   747→	return data, nil
   748→}
   749→
   750→// buildNodeQueries constructs APIQuery structures for node-based queries with scope transformation.
   751→//
   752→// Similar to buildQueries but operates on node lists rather than job resources.
   753→// Supports dynamic subcluster lookup when subCluster parameter is empty.
   754→//
   755→// Parameters:
   756→//   - cluster: Cluster name
   757→//   - subCluster: SubCluster name (empty = infer from node hostnames)
   758→//   - nodes: List of node hostnames
   759→//   - metrics: List of metric names
   760→//   - scopes: Requested metric scopes
   761→//   - resolution: Data resolution in seconds
   762→//
   763→// Returns:
   764→//   - []APIQuery: List of queries to execute
   765→//   - []schema.MetricScope: Assigned scope for each query
   766→//   - error: Returns error if topology lookup fails or unhandled scope combination
   767→func buildNodeQueries(
   768→	cluster string,
   769→	subCluster string,
   770→	nodes []string,
   771→	metrics []string,
   772→	scopes []schema.MetricScope,
   773→	resolution int64,
   774→) ([]APIQuery, []schema.MetricScope, error) {
   775→	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
   776→	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(nodes))
   777→
   778→	// Get Topol before loop if subCluster given
   779→	var subClusterTopol *schema.SubCluster
   780→	var scterr error
   781→	if subCluster != "" {
   782→		subClusterTopol, scterr = archive.GetSubCluster(cluster, subCluster)
   783→		if scterr != nil {
   784→			cclog.Errorf("could not load cluster %s subCluster %s topology: %s", cluster, subCluster, scterr.Error())
   785→			return nil, nil, scterr
   786→		}
   787→	}
   788→
   789→	for _, metric := range metrics {
   790→		mc := archive.GetMetricConfig(cluster, metric)
   791→		if mc == nil {
   792→			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
   793→			continue
   794→		}
   795→
   796→		// Skip if metric is removed for subcluster
   797→		if mc.SubClusters != nil && IsMetricRemovedForSubCluster(mc, subCluster) {
   798→			continue
   799→		}
   800→
   801→		// Avoid duplicates...
   802→		handledScopes := make([]schema.MetricScope, 0, 3)
   803→
   804→	nodeScopesLoop:
   805→		for _, requestedScope := range scopes {
   806→			nativeScope := mc.Scope
   807→
   808→			scope := nativeScope.Max(requestedScope)
   809→			for _, s := range handledScopes {
   810→				if scope == s {
   811→					continue nodeScopesLoop
   812→				}
   813→			}
   814→			handledScopes = append(handledScopes, scope)
   815→
   816→			for _, hostname := range nodes {
   817→
   818→				// If no subCluster given, get it by node
   819→				if subCluster == "" {
   820→					subClusterName, scnerr := archive.GetSubClusterByNode(cluster, hostname)
   821→					if scnerr != nil {
   822→						return nil, nil, scnerr
   823→					}
   824→					subClusterTopol, scterr = archive.GetSubCluster(cluster, subClusterName)
   825→					if scterr != nil {
   826→						return nil, nil, scterr
   827→					}
   828→				}
   829→
   830→				// Always full node hwthread id list, no partial queries expected
   831→				topology := subClusterTopol.Topology
   832→				acceleratorIds := topology.GetAcceleratorIDs()
   833→
   834→				// Moved check here if metric matches hardware specs
   835→				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
   836→					continue
   837→				}
   838→
   839→				scopeResults, ok := BuildScopeQueries(
   840→					nativeScope, requestedScope,
   841→					metric, hostname,
   842→					&topology, topology.Node, acceleratorIds,
   843→				)
   844→
   845→				if !ok {
   846→					return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
   847→				}
   848→
   849→				for _, sr := range scopeResults {
   850→					queries = append(queries, APIQuery{
   851→						Metric:     sr.Metric,
   852→						Hostname:   sr.Hostname,
   853→						Aggregate:  sr.Aggregate,
   854→						Type:       sr.Type,
   855→						TypeIds:    sr.TypeIds,
   856→						Resolution: resolution,
   857→					})
   858→					assignedScope = append(assignedScope, sr.Scope)
   859→				}
   860→			}
   861→		}
   862→	}
   863→
   864→	return queries, assignedScope, nil
   865→}
   866→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// This file implements high-level query functions for loading job metric data
// with automatic scope transformation and aggregation.
//
// Key Concepts:
//
// Metric Scopes: Metrics are collected at different granularities (native scope):
//   - HWThread: Per hardware thread
//   - Core: Per CPU core
//   - Socket: Per CPU socket
//   - MemoryDomain: Per memory domain (NUMA)
//   - Accelerator: Per GPU/accelerator
//   - Node: Per compute node
//
// Scope Transformation: The buildQueries functions transform between native scope
// and requested scope by:
//   - Aggregating finer-grained data (e.g., HWThread → Core → Socket → Node)
//   - Rejecting requests for finer granularity than available
//   - Handling special cases (e.g., Accelerator metrics)
//
// Query Building: Constructs APIQuery structures with proper selectors (Type, TypeIds)
// based on cluster topology and job resources.
package metricstore

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

type InternalMetricStore struct{}

var MetricStoreHandle *InternalMetricStore

// HealthCheck delegates to the internal MemoryStore's HealthCheck.
func (ccms *InternalMetricStore) HealthCheck(cluster string,
	nodes []string, metrics []string,
) (map[string]HealthCheckResult, error) {
	return GetMemoryStore().HealthCheck(cluster, nodes, metrics)
}

// TestLoadDataCallback allows tests to override LoadData behavior for testing purposes.
// When set to a non-nil function, LoadData will call this function instead of the default implementation.
var TestLoadDataCallback func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error)

// LoadData loads metric data for a specific job with automatic scope transformation.
//
// This is the primary function for retrieving job metric data. It handles:
//   - Building queries with scope transformation via buildQueries
//   - Fetching data from the metric store
//   - Organizing results by metric and scope
//   - Converting NaN statistics to 0 for JSON compatibility
//   - Partial error handling (returns data for successful queries even if some fail)
//
// Parameters:
//   - job: Job metadata including cluster, resources, and time range
//   - metrics: List of metric names to load
//   - scopes: Requested metric scopes (will be transformed to match native scopes)
//   - ctx: Context for cancellation (currently unused but reserved for future use)
//   - resolution: Data resolution in seconds (0 for native resolution)
//
// Returns:
//   - JobData: Map of metric → scope → JobMetric with time-series data and statistics
//   - Error: Returns error if query building or fetching fails, or partial error listing failed hosts
//
// Example:
//
//	jobData, err := LoadData(job, []string{"cpu_load", "mem_used"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 60)
func (ccms *InternalMetricStore) LoadData(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
	resolution int,
) (schema.JobData, error) {
	if TestLoadDataCallback != nil {
		return TestLoadDataCallback(job, metrics, scopes, ctx, resolution)
	}

	queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  true,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	jobData := make(schema.JobData)

	// Add safety check for potential index out of range errors
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(job.Cluster, metric)

		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, job.Cluster)
			continue
		}

		if _, ok := jobData[metric]; !ok {
			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = int(row[0].Resolution)
		}

		jobMetric, ok := jobData[metric][scope]
		if !ok {
			jobMetric = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			jobData[metric][scope] = jobMetric
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)

			SanitizeStats(&res.Avg, &res.Min, &res.Max)

			jobMetric.Series = append(jobMetric.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}

		// So that one can later check len(jobData):
		if len(jobMetric.Series) == 0 {
			delete(jobData[metric], scope)
			if len(jobData[metric]) == 0 {
				delete(jobData, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return jobData, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return jobData, nil
}

// buildQueries constructs APIQuery structures with automatic scope transformation for a job.
//
// This function implements the core scope transformation logic, handling all combinations of
// native metric scopes and requested scopes. It uses the cluster topology to determine which
// hardware IDs to include in each query.
//
// Scope Transformation Rules:
//   - If native scope >= requested scope: Aggregates data (Aggregate=true in APIQuery)
//   - If native scope < requested scope: Returns error (cannot increase granularity)
//   - Special handling for Accelerator scope (independent of CPU hierarchy)
//
// The function generates one or more APIQuery per (metric, scope, host) combination:
//   - For non-aggregated queries: One query with all relevant IDs
//   - For aggregated queries: May generate multiple queries (e.g., one per socket/core)
//
// Parameters:
//   - job: Job metadata including cluster, subcluster, and resource allocation
//   - metrics: List of metrics to query
//   - scopes: Requested scopes for each metric
//   - resolution: Data resolution in seconds
//
// Returns:
//   - []APIQuery: List of queries to execute
//   - []schema.MetricScope: Assigned scope for each query (after transformation)
//   - error: Returns error if topology lookup fails or unhandled scope combination encountered
func buildQueries(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int64,
) ([]APIQuery, []schema.MetricScope, error) {
	if len(job.Resources) == 0 {
		return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > no resources allocated for job %d", job.JobID)
	}

	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(job.Resources))
	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(job.Resources))

	subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster)
	if scerr != nil {
		return nil, nil, scerr
	}
	topology := subcluster.Topology

	for _, metric := range metrics {
		mc := archive.GetMetricConfig(job.Cluster, metric)
		if mc == nil {
			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, job.Cluster)
			continue
		}

		// Skip if metric is removed for subcluster
		if len(mc.SubClusters) != 0 && IsMetricRemovedForSubCluster(mc, job.SubCluster) {
			continue
		}

		// Avoid duplicates...
		handledScopes := make([]schema.MetricScope, 0, 3)

	scopesLoop:
		for _, requestedScope := range scopes {
			nativeScope := mc.Scope
			if nativeScope == schema.MetricScopeAccelerator && job.NumAcc == 0 {
				continue
			}

			scope := nativeScope.Max(requestedScope)
			for _, s := range handledScopes {
				if scope == s {
					continue scopesLoop
				}
			}
			handledScopes = append(handledScopes, scope)

			for _, host := range job.Resources {
				hwthreads := host.HWThreads
				if hwthreads == nil {
					hwthreads = topology.Node
				}

				scopeResults, ok := BuildScopeQueries(
					nativeScope, requestedScope,
					metric, host.Hostname,
					&topology, hwthreads, host.Accelerators,
				)

				if !ok {
					return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
				}

				for _, sr := range scopeResults {
					queries = append(queries, APIQuery{
						Metric:     sr.Metric,
						Hostname:   sr.Hostname,
						Aggregate:  sr.Aggregate,
						Type:       sr.Type,
						TypeIds:    sr.TypeIds,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, sr.Scope)
				}
			}
		}
	}

	return queries, assignedScope, nil
}

// LoadStats loads only metric statistics (avg/min/max) for a job at node scope.
//
// This is an optimized version of LoadData that fetches only statistics without
// time-series data, reducing bandwidth and memory usage. Always queries at node scope.
//
// Parameters:
//   - job: Job metadata
//   - metrics: List of metric names
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of metric → hostname → statistics
//   - Error on query building or fetching failure
func (ccms *InternalMetricStore) LoadStats(
	job *schema.Job,
	metrics []string,
	ctx context.Context,
) (map[string]map[string]schema.MetricStatistics, error) {
	// TODO(#166): Add scope parameter for analysis view accelerator normalization
	queries, _, err := buildQueries(job, metrics, []schema.MetricScope{schema.MetricScopeNode}, 0)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v: %s", job.JobID, metrics, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	stats := make(map[string]map[string]schema.MetricStatistics, len(metrics))
	for i, res := range resBody.Results {
		if i >= len(req.Queries) {
			cclog.Warnf("LoadStats: result index %d exceeds queries length %d", i, len(req.Queries))
			break
		}
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}
		query := req.Queries[i]
		metric := query.Metric
		data := res[0]
		if data.Error != nil {
			cclog.Warnf("fetching %s for node %s failed: %s", metric, query.Hostname, *data.Error)
			continue
		}

		metricdata, ok := stats[metric]
		if !ok {
			metricdata = make(map[string]schema.MetricStatistics, job.NumNodes)
			stats[metric] = metricdata
		}

		if data.Avg.IsNaN() || data.Min.IsNaN() || data.Max.IsNaN() {
			cclog.Warnf("fetching %s for node %s failed: one of avg/min/max is NaN", metric, query.Hostname)
			continue
		}

		metricdata[query.Hostname] = schema.MetricStatistics{
			Avg: float64(data.Avg),
			Min: float64(data.Min),
			Max: float64(data.Max),
		}
	}

	return stats, nil
}

// LoadScopedStats loads metric statistics for a job with scope-aware grouping.
//
// Similar to LoadStats but supports multiple scopes and returns statistics grouped
// by scope with hardware IDs (e.g., per-core, per-socket statistics).
//
// Parameters:
//   - job: Job metadata
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - ctx: Context (currently unused)
//
// Returns:
//   - ScopedJobStats: Map of metric → scope → []ScopedStats (with hostname and ID)
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadScopedStats(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
) (schema.ScopedJobStats, error) {
	queries, assignedScope, err := buildQueries(job, metrics, scopes, 0)
	if err != nil {
		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
		return nil, err
	}

	req := APIQueryRequest{
		Cluster:   job.Cluster,
		From:      job.StartTime,
		To:        job.StartTime + int64(job.Duration),
		Queries:   queries,
		WithStats: true,
		WithData:  false,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	scopedJobStats := make(schema.ScopedJobStats)

	for i, row := range resBody.Results {
		if len(row) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}
		query := req.Queries[i]
		metric := query.Metric
		scope := assignedScope[i]

		if _, ok := scopedJobStats[metric]; !ok {
			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
		}

		if _, ok := scopedJobStats[metric][scope]; !ok {
			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)

			SanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
				Hostname: query.Hostname,
				ID:       id,
				Data: &schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
			})
		}

		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
		if len(scopedJobStats[metric][scope]) == 0 {
			delete(scopedJobStats[metric], scope)
			if len(scopedJobStats[metric]) == 0 {
				delete(scopedJobStats, metric)
			}
		}
	}

	if len(errors) != 0 {
		/* Returns list for "partial errors" */
		return scopedJobStats, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}
	return scopedJobStats, nil
}

// LoadNodeData loads metric data for specific nodes in a cluster over a time range.
//
// Unlike LoadData which operates on job resources, this function queries arbitrary nodes
// directly. Useful for system monitoring and node status views.
//
// Parameters:
//   - cluster: Cluster name
//   - metrics: List of metric names
//   - nodes: List of node hostnames (nil = all nodes in cluster via ForAllNodes)
//   - scopes: Requested metric scopes (currently unused - always node scope)
//   - from, to: Time range
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of hostname → metric → []JobMetric
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadNodeData(
	cluster string,
	metrics, nodes []string,
	scopes []schema.MetricScope,
	from, to time.Time,
	ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
	req := APIQueryRequest{
		Cluster:   cluster,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	if nodes == nil {
		req.ForAllNodes = append(req.ForAllNodes, metrics...)
	} else {
		for _, node := range nodes {
			for _, metric := range metrics {
				req.Queries = append(req.Queries, APIQuery{
					Hostname:   node,
					Metric:     metric,
					Resolution: 0, // Default for Node Queries: Will return metric $Timestep Resolution
				})
			}
		}
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]map[string][]*schema.JobMetric)
	for i, res := range resBody.Results {
		if len(res) == 0 {
			// No Data Found For Metric, Logged in FetchData to Warn
			continue
		}

		var query APIQuery
		if resBody.Queries != nil {
			query = resBody.Queries[i]
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		qdata := res[0]
		if qdata.Error != nil {
			errors = append(errors, fmt.Sprintf("fetching %s for node %s failed: %s", metric, query.Hostname, *qdata.Error))
			continue
		}

		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
			continue
		}

		SanitizeStats(&qdata.Avg, &qdata.Min, &qdata.Max)

		hostdata, ok := data[query.Hostname]
		if !ok {
			hostdata = make(map[string][]*schema.JobMetric)
			data[query.Hostname] = hostdata
		}

		hostdata[metric] = append(hostdata[metric], &schema.JobMetric{
			Unit:     mc.Unit,
			Timestep: mc.Timestep,
			Series: []schema.Series{
				{
					Hostname: query.Hostname,
					Data:     qdata.Data,
					Statistics: schema.MetricStatistics{
						Avg: float64(qdata.Avg),
						Min: float64(qdata.Min),
						Max: float64(qdata.Max),
					},
				},
			},
		})
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// LoadNodeListData loads metric data for a list of nodes with full scope transformation support.
//
// This is the most flexible node data loading function, supporting arbitrary scopes and
// resolution. Uses buildNodeQueries for proper scope transformation based on topology.
//
// Parameters:
//   - cluster: Cluster name
//   - subCluster: SubCluster name (empty string to infer from node names)
//   - nodes: List of node hostnames
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - resolution: Data resolution in seconds
//   - from, to: Time range
//   - ctx: Context (currently unused)
//
// Returns:
//   - Map of hostname → JobData (metric → scope → JobMetric)
//   - Error or partial error listing failed queries
func (ccms *InternalMetricStore) LoadNodeListData(
	cluster, subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	from, to time.Time,
	ctx context.Context,
) (map[string]schema.JobData, error) {
	// Note: Order of node data is not guaranteed after this point
	queries, assignedScope, err := buildNodeQueries(cluster, subCluster, nodes, metrics, scopes, int64(resolution))
	if err != nil {
		cclog.Errorf("Error while building node queries for Cluster %s, SubCLuster %s, Metrics %v, Scopes %v: %s", cluster, subCluster, metrics, scopes, err.Error())
		return nil, err
	}

	// Verify assignment is correct - log any inconsistencies for debugging
	if len(queries) != len(assignedScope) {
		cclog.Errorf("Critical error: queries and assignedScope have different lengths after buildNodeQueries: %d vs %d",
			len(queries), len(assignedScope))
	}

	req := APIQueryRequest{
		Cluster:   cluster,
		Queries:   queries,
		From:      from.Unix(),
		To:        to.Unix(),
		WithStats: true,
		WithData:  true,
	}

	resBody, err := FetchData(req)
	if err != nil {
		cclog.Errorf("Error while fetching data : %s", err.Error())
		return nil, err
	}

	var errors []string
	data := make(map[string]schema.JobData)

	// Add safety check for index out of range issues
	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
		cclog.Warnf("Mismatch in query results count: queries=%d, results=%d, assignedScope=%d",
			len(req.Queries), len(resBody.Results), len(assignedScope))
		if len(resBody.Results) > len(req.Queries) {
			resBody.Results = resBody.Results[:len(req.Queries)]
		}
		if len(assignedScope) > len(req.Queries) {
			assignedScope = assignedScope[:len(req.Queries)]
		}
	}

	for i, row := range resBody.Results {
		var query APIQuery
		if resBody.Queries != nil {
			if i < len(resBody.Queries) {
				query = resBody.Queries[i]
			} else {
				cclog.Warnf("Index out of range prevented for resBody.Queries: i=%d, len=%d",
					i, len(resBody.Queries))
				continue
			}
		} else {
			query = req.Queries[i]
		}

		metric := query.Metric
		scope := assignedScope[i]
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("Metric config not found for %s on cluster %s", metric, cluster)
			continue
		}

		res := mc.Timestep
		if len(row) > 0 {
			res = int(row[0].Resolution)
		}

		// Init Nested Map Data Structures If Not Found
		hostData, ok := data[query.Hostname]
		if !ok {
			hostData = make(schema.JobData)
			data[query.Hostname] = hostData
		}

		metricData, ok := hostData[metric]
		if !ok {
			metricData = make(map[schema.MetricScope]*schema.JobMetric)
			data[query.Hostname][metric] = metricData
		}

		scopeData, ok := metricData[scope]
		if !ok {
			scopeData = &schema.JobMetric{
				Unit:     mc.Unit,
				Timestep: res,
				Series:   make([]schema.Series, 0),
			}
			data[query.Hostname][metric][scope] = scopeData
		}

		for ndx, res := range row {
			if res.Error != nil {
				/* Build list for "partial errors", if any */
				errors = append(errors, fmt.Sprintf("failed to fetch '%s' from host '%s': %s", query.Metric, query.Hostname, *res.Error))
				continue
			}

			id := ExtractTypeID(query.Type, query.TypeIds, ndx, query.Metric, query.Hostname)

			SanitizeStats(&res.Avg, &res.Min, &res.Max)

			scopeData.Series = append(scopeData.Series, schema.Series{
				Hostname: query.Hostname,
				ID:       id,
				Statistics: schema.MetricStatistics{
					Avg: float64(res.Avg),
					Min: float64(res.Min),
					Max: float64(res.Max),
				},
				Data: res.Data,
			})
		}
	}

	if len(errors) != 0 {
		/* Returns list of "partial errors" */
		return data, fmt.Errorf("METRICDATA/INTERNAL-CCMS > Errors: %s", strings.Join(errors, ", "))
	}

	return data, nil
}

// buildNodeQueries constructs APIQuery structures for node-based queries with scope transformation.
//
// Similar to buildQueries but operates on node lists rather than job resources.
// Supports dynamic subcluster lookup when subCluster parameter is empty.
//
// Parameters:
//   - cluster: Cluster name
//   - subCluster: SubCluster name (empty = infer from node hostnames)
//   - nodes: List of node hostnames
//   - metrics: List of metric names
//   - scopes: Requested metric scopes
//   - resolution: Data resolution in seconds
//
// Returns:
//   - []APIQuery: List of queries to execute
//   - []schema.MetricScope: Assigned scope for each query
//   - error: Returns error if topology lookup fails or unhandled scope combination
func buildNodeQueries(
	cluster string,
	subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int64,
) ([]APIQuery, []schema.MetricScope, error) {
	queries := make([]APIQuery, 0, len(metrics)*len(scopes)*len(nodes))
	assignedScope := make([]schema.MetricScope, 0, len(metrics)*len(scopes)*len(nodes))

	// Get Topol before loop if subCluster given
	var subClusterTopol *schema.SubCluster
	var scterr error
	if subCluster != "" {
		subClusterTopol, scterr = archive.GetSubCluster(cluster, subCluster)
		if scterr != nil {
			cclog.Errorf("could not load cluster %s subCluster %s topology: %s", cluster, subCluster, scterr.Error())
			return nil, nil, scterr
		}
	}

	for _, metric := range metrics {
		mc := archive.GetMetricConfig(cluster, metric)
		if mc == nil {
			cclog.Warnf("metric '%s' is not specified for cluster '%s'", metric, cluster)
			continue
		}

		// Skip if metric is removed for subcluster
		if mc.SubClusters != nil && IsMetricRemovedForSubCluster(mc, subCluster) {
			continue
		}

		// Avoid duplicates...
		handledScopes := make([]schema.MetricScope, 0, 3)

	nodeScopesLoop:
		for _, requestedScope := range scopes {
			nativeScope := mc.Scope

			scope := nativeScope.Max(requestedScope)
			for _, s := range handledScopes {
				if scope == s {
					continue nodeScopesLoop
				}
			}
			handledScopes = append(handledScopes, scope)

			for _, hostname := range nodes {

				// If no subCluster given, get it by node
				if subCluster == "" {
					subClusterName, scnerr := archive.GetSubClusterByNode(cluster, hostname)
					if scnerr != nil {
						return nil, nil, scnerr
					}
					subClusterTopol, scterr = archive.GetSubCluster(cluster, subClusterName)
					if scterr != nil {
						return nil, nil, scterr
					}
				}

				// Always full node hwthread id list, no partial queries expected
				topology := subClusterTopol.Topology
				acceleratorIds := topology.GetAcceleratorIDs()

				// Moved check here if metric matches hardware specs
				if nativeScope == schema.MetricScopeAccelerator && len(acceleratorIds) == 0 {
					continue
				}

				scopeResults, ok := BuildScopeQueries(
					nativeScope, requestedScope,
					metric, hostname,
					&topology, topology.Node, acceleratorIds,
				)

				if !ok {
					return nil, nil, fmt.Errorf("METRICDATA/INTERNAL-CCMS > unsupported scope transformation: native-scope=%s, requested-scope=%s", nativeScope, requestedScope)
				}

				for _, sr := range scopeResults {
					queries = append(queries, APIQuery{
						Metric:     sr.Metric,
						Hostname:   sr.Hostname,
						Aggregate:  sr.Aggregate,
						Type:       sr.Type,
						TypeIds:    sr.TypeIds,
						Resolution: resolution,
					})
					assignedScope = append(assignedScope, sr.Scope)
				}
			}
		}
	}

	return queries, assignedScope, nil
}


Now let me read the resolvers file to see the current signatures.


501:func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope, resolution *int) ([]*model.JobMetricWithName, error) {
502-	if resolution == nil { // Load from Config
503-		if config.Keys.EnableResampling != nil {
504-			defaultRes := slices.Max(config.Keys.EnableResampling.Resolutions)
505-			resolution = &defaultRes
506-		} else { // Set 0 (Loads configured metric timestep)
--
875:func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, subCluster string, stateFilter string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) (*model.NodesResultList, error) {
876-	if resolution == nil { // Load from Config
877-		if config.Keys.EnableResampling != nil {
878-			defaultRes := slices.Max(config.Keys.EnableResampling.Resolutions)
879-			resolution = &defaultRes
880-		} else { // Set 0 (Loads configured metric timestep)


   490→
   491→	if user := repository.GetUserFromContext(ctx); user != nil &&
   492→		job.User != user.Username &&
   493→		user.HasNotRoles([]schema.Role{schema.RoleAdmin, schema.RoleSupport, schema.RoleManager}) {
   494→		return nil, errors.New("you are not allowed to see this job")
   495→	}
   496→
   497→	return job, nil
   498→}
   499→
   500→// JobMetrics is the resolver for the jobMetrics field.
   501→func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope, resolution *int) ([]*model.JobMetricWithName, error) {
   502→	if resolution == nil { // Load from Config
   503→		if config.Keys.EnableResampling != nil {
   504→			defaultRes := slices.Max(config.Keys.EnableResampling.Resolutions)
   505→			resolution = &defaultRes
   506→		} else { // Set 0 (Loads configured metric timestep)
   507→			defaultRes := 0
   508→			resolution = &defaultRes
   509→		}
   510→	}
   511→
   512→	job, err := r.Query().Job(ctx, id)
   513→	if err != nil {
   514→		cclog.Warn("Error while querying job for metrics")
   515→		return nil, err
   516→	}
   517→
   518→	data, err := metricdispatch.LoadData(job, metrics, scopes, ctx, *resolution)
   519→	if err != nil {

	if user := repository.GetUserFromContext(ctx); user != nil &&
		job.User != user.Username &&
		user.HasNotRoles([]schema.Role{schema.RoleAdmin, schema.RoleSupport, schema.RoleManager}) {
		return nil, errors.New("you are not allowed to see this job")
	}

	return job, nil
}

// JobMetrics is the resolver for the jobMetrics field.
func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope, resolution *int) ([]*model.JobMetricWithName, error) {
	if resolution == nil { // Load from Config
		if config.Keys.EnableResampling != nil {
			defaultRes := slices.Max(config.Keys.EnableResampling.Resolutions)
			resolution = &defaultRes
		} else { // Set 0 (Loads configured metric timestep)
			defaultRes := 0
			resolution = &defaultRes
		}
	}

	job, err := r.Query().Job(ctx, id)
	if err != nil {
		cclog.Warn("Error while querying job for metrics")
		return nil, err
	}

	data, err := metricdispatch.LoadData(job, metrics, scopes, ctx, *resolution)
	if err != nil {


   860→				host.Metrics = append(host.Metrics, &model.JobMetricWithName{
   861→					Name:   metric,
   862→					Scope:  schema.MetricScopeNode,
   863→					Metric: scopedMetric,
   864→				})
   865→			}
   866→		}
   867→
   868→		nodeMetrics = append(nodeMetrics, host)
   869→	}
   870→
   871→	return nodeMetrics, nil
   872→}
   873→
   874→// NodeMetricsList is the resolver for the nodeMetricsList field.
   875→func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, subCluster string, stateFilter string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) (*model.NodesResultList, error) {
   876→	if resolution == nil { // Load from Config
   877→		if config.Keys.EnableResampling != nil {
   878→			defaultRes := slices.Max(config.Keys.EnableResampling.Resolutions)
   879→			resolution = &defaultRes
   880→		} else { // Set 0 (Loads configured metric timestep)
   881→			defaultRes := 0
   882→			resolution = &defaultRes
   883→		}
   884→	}
   885→
   886→	user := repository.GetUserFromContext(ctx)
   887→	if user != nil && !user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) {
   888→		return nil, errors.New("you need to be administrator or support staff for this query")
   889→	}
   890→
   891→	nodeRepo := repository.GetNodeRepository()
   892→	// nodes -> array hostname
   893→	nodes, nodeStateMap, metricHealthMap, countNodes, hasNextPage, nerr := nodeRepo.GetNodesForList(ctx, cluster, subCluster, stateFilter, nodeFilter, page)
   894→	if nerr != nil {
   895→		return nil, errors.New("could not retrieve node list required for resolving NodeMetricsList")
   896→	}
   897→
   898→	if metrics == nil {
   899→		for _, mc := range archive.GetCluster(cluster).MetricConfig {

				host.Metrics = append(host.Metrics, &model.JobMetricWithName{
					Name:   metric,
					Scope:  schema.MetricScopeNode,
					Metric: scopedMetric,
				})
			}
		}

		nodeMetrics = append(nodeMetrics, host)
	}

	return nodeMetrics, nil
}

// NodeMetricsList is the resolver for the nodeMetricsList field.
func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, subCluster string, stateFilter string, nodeFilter string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time, page *model.PageRequest, resolution *int) (*model.NodesResultList, error) {
	if resolution == nil { // Load from Config
		if config.Keys.EnableResampling != nil {
			defaultRes := slices.Max(config.Keys.EnableResampling.Resolutions)
			resolution = &defaultRes
		} else { // Set 0 (Loads configured metric timestep)
			defaultRes := 0
			resolution = &defaultRes
		}
	}

	user := repository.GetUserFromContext(ctx)
	if user != nil && !user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) {
		return nil, errors.New("you need to be administrator or support staff for this query")
	}

	nodeRepo := repository.GetNodeRepository()
	// nodes -> array hostname
	nodes, nodeStateMap, metricHealthMap, countNodes, hasNextPage, nerr := nodeRepo.GetNodesForList(ctx, cluster, subCluster, stateFilter, nodeFilter, page)
	if nerr != nil {
		return nil, errors.New("could not retrieve node list required for resolving NodeMetricsList")
	}

	if metrics == nil {
		for _, mc := range archive.GetCluster(cluster).MetricConfig {


Let me see more of the NodeMetricsList resolver to find where LoadNodeListData is called.


   898→	if metrics == nil {
   899→		for _, mc := range archive.GetCluster(cluster).MetricConfig {
   900→			metrics = append(metrics, mc.Name)
   901→		}
   902→	}
   903→
   904→	// data -> map hostname:jobdata
   905→	data, err := metricdispatch.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, *resolution, from, to, ctx)
   906→	if err != nil {
   907→		cclog.Warn("error while loading node data (Resolver.NodeMetricsList")
   908→		return nil, err
   909→	}
   910→
   911→	nodeMetricsList := make([]*model.NodeMetrics, 0, len(data))
   912→	for _, hostname := range nodes {
   913→		host := &model.NodeMetrics{
   914→			Host:         hostname,
   915→			NodeState:    nodeStateMap[hostname],
   916→			MetricHealth: metricHealthMap[hostname],
   917→			Metrics:      make([]*model.JobMetricWithName, 0),
   918→		}
   919→		host.SubCluster, err = archive.GetSubClusterByNode(cluster, hostname)
   920→		if err != nil {
   921→			cclog.Warnf("error in nodeMetrics resolver: %s", err)
   922→		}
   923→
   924→		for metric, scopedMetrics := range data[hostname] {
   925→			for scope, scopedMetric := range scopedMetrics {
   926→				host.Metrics = append(host.Metrics, &model.JobMetricWithName{
   927→					Name:   metric,
   928→					Scope:  scope,
   929→					Metric: scopedMetric,
   930→				})
   931→			}
   932→		}
   933→
   934→		nodeMetricsList = append(nodeMetricsList, host)
   935→	}
   936→
   937→	nodeMetricsListResult := &model.NodesResultList{

	if metrics == nil {
		for _, mc := range archive.GetCluster(cluster).MetricConfig {
			metrics = append(metrics, mc.Name)
		}
	}

	// data -> map hostname:jobdata
	data, err := metricdispatch.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, *resolution, from, to, ctx)
	if err != nil {
		cclog.Warn("error while loading node data (Resolver.NodeMetricsList")
		return nil, err
	}

	nodeMetricsList := make([]*model.NodeMetrics, 0, len(data))
	for _, hostname := range nodes {
		host := &model.NodeMetrics{
			Host:         hostname,
			NodeState:    nodeStateMap[hostname],
			MetricHealth: metricHealthMap[hostname],
			Metrics:      make([]*model.JobMetricWithName, 0),
		}
		host.SubCluster, err = archive.GetSubClusterByNode(cluster, hostname)
		if err != nil {
			cclog.Warnf("error in nodeMetrics resolver: %s", err)
		}

		for metric, scopedMetrics := range data[hostname] {
			for scope, scopedMetric := range scopedMetrics {
				host.Metrics = append(host.Metrics, &model.JobMetricWithName{
					Name:   metric,
					Scope:  scope,
					Metric: scopedMetric,
				})
			}
		}

		nodeMetricsList = append(nodeMetricsList, host)
	}

	nodeMetricsListResult := &model.NodesResultList{


     1→// Code generated by github.com/99designs/gqlgen, DO NOT EDIT.
     2→
     3→package model
     4→
     5→import (
     6→	"bytes"
     7→	"fmt"
     8→	"io"
     9→	"strconv"
    10→	"time"
    11→
    12→	"github.com/ClusterCockpit/cc-backend/internal/config"
    13→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    14→)
    15→
    16→type ClusterMetricWithName struct {
    17→	Name     string         `json:"name"`
    18→	Unit     *schema.Unit   `json:"unit,omitempty"`
    19→	Timestep int            `json:"timestep"`
    20→	Data     []schema.Float `json:"data"`
    21→}
    22→
    23→type ClusterMetrics struct {
    24→	NodeCount int                      `json:"nodeCount"`
    25→	Metrics   []*ClusterMetricWithName `json:"metrics"`
    26→}
    27→
    28→type Count struct {
    29→	Name  string `json:"name"`
    30→	Count int    `json:"count"`
    31→}
    32→
    33→type EnergyFootprintValue struct {
    34→	Hardware string  `json:"hardware"`
    35→	Metric   string  `json:"metric"`
    36→	Value    float64 `json:"value"`
    37→}
    38→
    39→type FloatRange struct {
    40→	From float64 `json:"from"`
    41→	To   float64 `json:"to"`
    42→}
    43→
    44→type FootprintValue struct {
    45→	Name  string  `json:"name"`
    46→	Stat  string  `json:"stat"`
    47→	Value float64 `json:"value"`
    48→}
    49→
    50→type Footprints struct {

// Code generated by github.com/99designs/gqlgen, DO NOT EDIT.

package model

import (
	"bytes"
	"fmt"
	"io"
	"strconv"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

type ClusterMetricWithName struct {
	Name     string         `json:"name"`
	Unit     *schema.Unit   `json:"unit,omitempty"`
	Timestep int            `json:"timestep"`
	Data     []schema.Float `json:"data"`
}

type ClusterMetrics struct {
	NodeCount int                      `json:"nodeCount"`
	Metrics   []*ClusterMetricWithName `json:"metrics"`
}

type Count struct {
	Name  string `json:"name"`
	Count int    `json:"count"`
}

type EnergyFootprintValue struct {
	Hardware string  `json:"hardware"`
	Metric   string  `json:"metric"`
	Value    float64 `json:"value"`
}

type FloatRange struct {
	From float64 `json:"from"`
	To   float64 `json:"to"`
}

type FootprintValue struct {
	Name  string  `json:"name"`
	Stat  string  `json:"stat"`
	Value float64 `json:"value"`
}

type Footprints struct {


Now I have a clear picture. Let me also check the existing `gqlgen.yml` to understand model mappings, and check the `LoadNodeListData` call in `metricdispatch/dataLoader.go`.


No files found


404:// LoadNodeListData retrieves time-series metric data for multiple nodes within a time range,
409:func LoadNodeListData(
431:	data, err := ms.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, resolution, from, to, ctx)


Good. Now I have the full picture. Let me implement this in stages. I'll start with Part 1 (cc-lib changes) and Part 2 (cc-backend changes) in parallel where possible.

## Part 1: cc-lib changes


Edit

func SetMinimumRequiredPoints(setVal int) {
	MinimumRequiredPoints = setVal
}

// ResamplerFunc is the signature shared by all resampler algorithms.
type ResamplerFunc func(data []schema.Float, oldFrequency int64, newFrequency int64) ([]schema.Float, int64, error)

// GetResampler returns the resampler function for the given name.
// Valid names: "lttb" (default), "average", "simple". Empty string returns LTTB.
func GetResampler(name string) (ResamplerFunc, error) {
	switch name {
	case "", "lttb":
		return LargestTriangleThreeBucket, nil
	case "average":
		return AverageResampler, nil
	case "simple":
		return SimpleResampler, nil
	default:
		return nil, fmt.Errorf("unknown resampler algorithm: %s", name)
	}
}

// AverageResampler performs RRDTool-style average consolidation by dividing data into
// fixed-size buckets and computing the arithmetic mean of all valid points in each bucket.
//
// This produces scientifically accurate averages over time intervals, unlike LTTB which
// preserves visual shape, or SimpleResampler which decimates. NaN values are skipped;
// if all values in a bucket are NaN, the output for that bucket is NaN.
//
// Parameters:
//   - data: input time-series data points
//   - oldFrequency: original sampling frequency
//   - newFrequency: target sampling frequency (must be a multiple of oldFrequency)
//
// Returns:
//   - Averaged data slice
//   - Actual frequency used
//   - Error if newFrequency is not a multiple of oldFrequency
func AverageResampler(data []schema.Float, oldFrequency int64, newFrequency int64) ([]schema.Float, int64, error) {
	newDataLength, step, err := validateFrequency(len(data), oldFrequency, newFrequency)
	if err != nil {
		return nil, 0, err
	}
	if newDataLength == -1 {
		return data, oldFrequency, nil
	}

	newData := make([]schema.Float, newDataLength)
	for i := range newDataLength {
		sum := 0.0
		count := 0
		base := i * step
		for j := range step {
			v := float64(data[base+j])
			if !math.IsNaN(v) {
				sum += v
				count++
			}
		}
		if count == 0 {
			newData[i] = schema.NaN
		} else {
			newData[i] = schema.Float(sum / float64(count))
		}
	}

	return newData, newFrequency, nil
}

func SetMinimumRequiredPoints(setVal int) {
	MinimumRequiredPoints = setVal
}