func LoadData(job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
	resolution int,
	resampleAlgo string,
) (schema.JobData, error) {
	data := cache.Get(cacheKey(job, metrics, scopes, resolution, resampleAlgo), func() (_ any, ttl time.Duration, size int) {

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricdispatch provides a unified interface for loading and caching job metric data.
//
// This package serves as a central dispatcher that routes metric data requests to the appropriate
// backend based on job state. For running jobs, data is fetched from the metric store (e.g., cc-metric-store).
// For completed jobs, data is retrieved from the file-based job archive.
//
// # Key Features
//
//   - Automatic backend selection based on job state (running vs. archived)
//   - LRU cache for performance optimization (128 MB default cache size)
//   - Data resampling using Largest Triangle Three Bucket algorithm for archived data
//   - Automatic statistics series generation for jobs with many nodes
//   - Support for scoped metrics (node, socket, accelerator, core)
//
// # Cache Behavior
//
// Cached data has different TTL (time-to-live) values depending on job state:
//   - Running jobs: 2 minutes (data changes frequently)
//   - Completed jobs: 5 hours (data is static)
//
// The cache key is based on job ID, state, requested metrics, scopes, and resolution.
//
// # Usage
//
// The primary entry point is LoadData, which automatically handles both running and archived jobs:
//
//	jobData, err := metricdispatch.LoadData(job, metrics, scopes, ctx, resolution)
//	if err != nil {
//	    // Handle error
//	}
//
// For statistics only, use LoadJobStats, LoadScopedJobStats, or LoadAverages depending on the required format.
package metricdispatch

import (
	"context"
	"fmt"
	"math"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/lrucache"
	"github.com/ClusterCockpit/cc-lib/v2/resampler"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// cache is an LRU cache with 128 MB capacity for storing loaded job metric data.
// The cache reduces load on both the metric store and archive backends.
var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024)

// cacheKey generates a unique cache key for a job's metric data based on job ID, state,
// requested metrics, scopes, and resolution. Duration and StartTime are intentionally excluded
// because job.ID is more unique and the cache TTL ensures entries don't persist indefinitely.
func cacheKey(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	resampleAlgo string,
) string {
	return fmt.Sprintf("%d(%s):[%v],[%v]-%d-%s",
		*job.ID, job.State, metrics, scopes, resolution, resampleAlgo)
}

// LoadData retrieves metric data for a job from the appropriate backend (memory store for running jobs,
// archive for completed jobs) and applies caching, resampling, and statistics generation as needed.
//
// For running jobs or when archive is disabled, data is fetched from the metric store.
// For completed archived jobs, data is loaded from the job archive and resampled if needed.
//
// Parameters:
//   - job: The job for which to load metric data
//   - metrics: List of metric names to load (nil loads all metrics for the cluster)
//   - scopes: Metric scopes to include (nil defaults to node scope)
//   - ctx: Context for cancellation and timeouts
//   - resolution: Target number of data points for resampling (only applies to archived data)
//
// Returns the loaded job data and any error encountered. For partial errors (some metrics failed),
// the function returns the successfully loaded data with a warning logged.
func LoadData(job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
	resolution int,
) (schema.JobData, error) {
	data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ any, ttl time.Duration, size int) {
		var jd schema.JobData
		var err error

		if job.State == schema.JobStateRunning ||
			job.MonitoringStatus == schema.MonitoringStatusRunningOrArchiving {

			ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
			if err != nil {
				cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
					job.Cluster, job.SubCluster, err.Error())
				return err, 0, 0
			}

			if scopes == nil {
				scopes = append(scopes, schema.MetricScopeNode)
			}

			if metrics == nil {
				cluster := archive.GetCluster(job.Cluster)
				for _, mc := range cluster.MetricConfig {
					metrics = append(metrics, mc.Name)
				}
			}

			jd, err = ms.LoadData(job, metrics, scopes, ctx, resolution)
			if err != nil {
				if len(jd) != 0 {
					cclog.Warnf("partial error loading metrics from store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
						job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
				} else {
					cclog.Warnf("failed to load job data from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
						job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
					return err, 0, 0
				}
			}
			size = jd.Size()
		} else {
			var jdTemp schema.JobData
			jdTemp, err = archive.GetHandle().LoadJobData(job)
			if err != nil {
				cclog.Warnf("failed to load job data from archive for job %d (user: %s, project: %s, cluster: %s-%s): %s",
					job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
				return err, 0, 0
			}

			jd = deepCopy(jdTemp)

			// Resample archived data using Largest Triangle Three Bucket algorithm to reduce data points
			// to the requested resolution, improving transfer performance and client-side rendering.
			for _, v := range jd {
				for _, v_ := range v {
					timestep := int64(0)
					for i := 0; i < len(v_.Series); i += 1 {
						v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))
						if err != nil {
							return err, 0, 0
						}
					}
					v_.Timestep = int(timestep)
				}
			}

			// Filter job data to only include requested metrics and scopes, avoiding unnecessary data transfer.
			if metrics != nil || scopes != nil {
				if metrics == nil {
					metrics = make([]string, 0, len(jd))
					for k := range jd {
						metrics = append(metrics, k)
					}
				}

				res := schema.JobData{}
				for _, metric := range metrics {
					if perscope, ok := jd[metric]; ok {
						if len(perscope) > 1 {
							subset := make(map[schema.MetricScope]*schema.JobMetric)
							for _, scope := range scopes {
								if jm, ok := perscope[scope]; ok {
									subset[scope] = jm
								}
							}

							if len(subset) > 0 {
								perscope = subset
							}
						}

						res[metric] = perscope
					}
				}
				jd = res
			}
			size = jd.Size()
		}

		ttl = 5 * time.Hour
		if job.State == schema.JobStateRunning {
			ttl = 2 * time.Minute
		}

		// Generate statistics series for jobs with many nodes to enable min/median/max graphs
		// instead of overwhelming the UI with individual node lines. Note that newly calculated
		// statistics use min/median/max, while archived statistics may use min/mean/max.
		const maxSeriesSize int = 8
		for _, scopes := range jd {
			for _, jm := range scopes {
				if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
					continue
				}

				jm.AddStatisticsSeries()
			}
		}

		nodeScopeRequested := false
		for _, scope := range scopes {
			if scope == schema.MetricScopeNode {
				nodeScopeRequested = true
			}
		}

		if nodeScopeRequested {
			jd.AddNodeScope("flops_any")
			jd.AddNodeScope("mem_bw")
		}

		// Round Resulting Stat Values
		jd.RoundMetricStats()

		return jd, ttl, size
	})

	if err, ok := data.(error); ok {
		cclog.Errorf("error in cached dataset for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	return data.(schema.JobData), nil
}

// LoadAverages computes average values for the specified metrics across all nodes of a job.
// For running jobs, it loads statistics from the metric store. For completed jobs, it uses
// the pre-calculated averages from the job archive. The results are appended to the data slice.
func LoadAverages(
	job *schema.Job,
	metrics []string,
	data [][]schema.Float,
	ctx context.Context,
) error {
	if job.State != schema.JobStateRunning {
		return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here?
	}

	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			job.Cluster, job.SubCluster, err.Error())
		return err
	}

	stats, err := ms.LoadStats(job, metrics, ctx)
	if err != nil {
		cclog.Warnf("failed to load statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
		return err
	}

	for i, m := range metrics {
		nodes, ok := stats[m]
		if !ok {
			data[i] = append(data[i], schema.NaN)
			continue
		}

		sum := 0.0
		for _, node := range nodes {
			sum += node.Avg
		}
		data[i] = append(data[i], schema.Float(sum))
	}

	return nil
}

// LoadScopedJobStats retrieves job statistics organized by metric scope (node, socket, core, accelerator).
// For running jobs, statistics are computed from the metric store. For completed jobs, pre-calculated
// statistics are loaded from the job archive.
func LoadScopedJobStats(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
) (schema.ScopedJobStats, error) {
	if job.State != schema.JobStateRunning {
		return archive.LoadScopedStatsFromArchive(job, metrics, scopes)
	}

	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			job.Cluster, job.SubCluster, err.Error())
		return nil, err
	}

	scopedStats, err := ms.LoadScopedStats(job, metrics, scopes, ctx)
	if err != nil {
		cclog.Warnf("failed to load scoped statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
		return nil, err
	}

	// Round Resulting Stat Values
	scopedStats.RoundScopedMetricStats()

	return scopedStats, nil
}

// LoadJobStats retrieves aggregated statistics (min/avg/max) for each requested metric across all job nodes.
// For running jobs, statistics are computed from the metric store. For completed jobs, pre-calculated
// statistics are loaded from the job archive.
func LoadJobStats(
	job *schema.Job,
	metrics []string,
	ctx context.Context,
) (map[string]schema.MetricStatistics, error) {
	if job.State != schema.JobStateRunning {
		return archive.LoadStatsFromArchive(job, metrics)
	}

	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			job.Cluster, job.SubCluster, err.Error())
		return nil, err
	}

	data := make(map[string]schema.MetricStatistics, len(metrics))

	stats, err := ms.LoadStats(job, metrics, ctx)
	if err != nil {
		cclog.Warnf("failed to load statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
		return data, err
	}

	for _, m := range metrics {
		sum, avg, min, max := 0.0, 0.0, 0.0, 0.0
		nodes, ok := stats[m]
		if !ok {
			data[m] = schema.MetricStatistics{Min: min, Avg: avg, Max: max}
			continue
		}

		for _, node := range nodes {
			sum += node.Avg
			min = math.Min(min, node.Min)
			max = math.Max(max, node.Max)
		}

		data[m] = schema.MetricStatistics{
			Avg: (math.Round((sum/float64(job.NumNodes))*100) / 100),
			Min: (math.Round(min*100) / 100),
			Max: (math.Round(max*100) / 100),
		}
	}

	return data, nil
}

// LoadNodeData retrieves metric data for specific nodes in a cluster within a time range.
// This is used for node monitoring views and system status pages. Data is always fetched from
// the metric store (not the archive) since it's for current/recent node status monitoring.
//
// Returns a nested map structure: node -> metric -> scoped data.
// FIXME: Add support for subcluster specific cc-metric-stores
func LoadNodeData(
	cluster string,
	metrics, nodes []string,
	scopes []schema.MetricScope,
	from, to time.Time,
	ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
	if metrics == nil {
		for _, m := range archive.GetCluster(cluster).MetricConfig {
			metrics = append(metrics, m.Name)
		}
	}

	ms, err := GetMetricDataRepo(cluster, "")
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s: %s",
			cluster, err.Error())
		return nil, err
	}

	data, err := ms.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
	if err != nil {
		if len(data) != 0 {
			cclog.Warnf("partial error loading node data from metric store for cluster %s: %s", cluster, err.Error())
		} else {
			cclog.Warnf("failed to load node data from metric store for cluster %s: %s", cluster, err.Error())
			return nil, err
		}
	}

	if data == nil {
		return nil, fmt.Errorf("metric store for cluster '%s' does not support node data queries", cluster)
	}

	return data, nil
}

// LoadNodeListData retrieves time-series metric data for multiple nodes within a time range,
// with optional resampling and automatic statistics generation for large datasets.
// This is used for comparing multiple nodes or displaying node status over time.
//
// Returns a map of node names to their job-like metric data structures.
func LoadNodeListData(
	cluster, subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	from, to time.Time,
	ctx context.Context,
) (map[string]schema.JobData, error) {
	if metrics == nil {
		for _, m := range archive.GetCluster(cluster).MetricConfig {
			metrics = append(metrics, m.Name)
		}
	}

	ms, err := GetMetricDataRepo(cluster, subCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			cluster, subCluster, err.Error())
		return nil, err
	}

	data, err := ms.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, resolution, from, to, ctx)
	if err != nil {
		if len(data) != 0 {
			cclog.Warnf("partial error loading node list data from metric store for cluster %s, subcluster %s: %s",
				cluster, subCluster, err.Error())
		} else {
			cclog.Warnf("failed to load node list data from metric store for cluster %s, subcluster %s: %s",
				cluster, subCluster, err.Error())
			return nil, err
		}
	}

	// Generate statistics series for datasets with many series to improve visualization performance.
	// Statistics are calculated as min/median/max.
	const maxSeriesSize int = 8
	for _, jd := range data {
		for _, scopes := range jd {
			for _, jm := range scopes {
				if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
					continue
				}
				jm.AddStatisticsSeries()
			}
		}
	}

	if data == nil {
		return nil, fmt.Errorf("metric store for cluster '%s' does not support node list queries", cluster)
	}

	return data, nil
}

// deepCopy creates a deep copy of JobData to prevent cache corruption when modifying
// archived data (e.g., during resampling). This ensures the cached archive data remains
// immutable while allowing per-request transformations.
func deepCopy(source schema.JobData) schema.JobData {
	result := make(schema.JobData, len(source))

	for metricName, scopeMap := range source {
		result[metricName] = make(map[schema.MetricScope]*schema.JobMetric, len(scopeMap))

		for scope, jobMetric := range scopeMap {
			result[metricName][scope] = copyJobMetric(jobMetric)
		}
	}

	return result
}

func copyJobMetric(src *schema.JobMetric) *schema.JobMetric {
	dst := &schema.JobMetric{
		Timestep: src.Timestep,
		Unit:     src.Unit,
		Series:   make([]schema.Series, len(src.Series)),
	}

	for i := range src.Series {
		dst.Series[i] = copySeries(&src.Series[i])
	}

	if src.StatisticsSeries != nil {
		dst.StatisticsSeries = copyStatisticsSeries(src.StatisticsSeries)
	}

	return dst
}

func copySeries(src *schema.Series) schema.Series {
	dst := schema.Series{
		Hostname:   src.Hostname,
		ID:         src.ID,
		Statistics: src.Statistics,
		Data:       make([]schema.Float, len(src.Data)),
	}

	copy(dst.Data, src.Data)
	return dst
}

func copyStatisticsSeries(src *schema.StatsSeries) *schema.StatsSeries {
	dst := &schema.StatsSeries{
		Min:    make([]schema.Float, len(src.Min)),
		Mean:   make([]schema.Float, len(src.Mean)),
		Median: make([]schema.Float, len(src.Median)),
		Max:    make([]schema.Float, len(src.Max)),
	}

	copy(dst.Min, src.Min)
	copy(dst.Mean, src.Mean)
	copy(dst.Median, src.Median)
	copy(dst.Max, src.Max)

	if len(src.Percentiles) > 0 {
		dst.Percentiles = make(map[int][]schema.Float, len(src.Percentiles))
		for percentile, values := range src.Percentiles {
			dst.Percentiles[percentile] = make([]schema.Float, len(values))
			copy(dst.Percentiles[percentile], values)
		}
	}

	return dst
}

-  data := cache.Get(cacheKey(job, metrics, scopes, resolution), func() (_ any, ttl time.Duration, size int) {

+  data := cache.Get(cacheKey(job, metrics, scopes, resolution, resampleAlgo), func() (_ any, ttl time.Duration, size int) {


Now update the resampler call in LoadData to use GetResampler:


Edit

		// Resample archived data using Largest Triangle Three Bucket algorithm to reduce data points
			// to the requested resolution, improving transfer performance and client-side rendering.
			for _, v := range jd {
				for _, v_ := range v {
					timestep := int64(0)
					for i := 0; i < len(v_.Series); i += 1 {
						v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))
						if err != nil {
							return err, 0, 0
						}
					}
					v_.Timestep = int(timestep)
				}
			}

		// Resample archived data to reduce data points to the requested resolution,
			// improving transfer performance and client-side rendering.
			resampleFn, rfErr := resampler.GetResampler(resampleAlgo)
			if rfErr != nil {
				return rfErr, 0, 0
			}
			for _, v := range jd {
				for _, v_ := range v {
					timestep := int64(0)
					for i := 0; i < len(v_.Series); i += 1 {
						v_.Series[i].Data, timestep, err = resampleFn(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))
						if err != nil {
							return err, 0, 0
						}
					}
					v_.Timestep = int(timestep)
				}
			}


The file /Users/jan/tmp/cc-backend/internal/metricdispatch/dataLoader.go has been updated successfully.

		// Resample archived data to reduce data points to the requested resolution,
			// improving transfer performance and client-side rendering.
			resampleFn, rfErr := resampler.GetResampler(resampleAlgo)
			if rfErr != nil {
				return rfErr, 0, 0
			}
			for _, v := range jd {
				for _, v_ := range v {
					timestep := int64(0)
					for i := 0; i < len(v_.Series); i += 1 {
						v_.Series[i].Data, timestep, err = resampleFn(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))
						if err != nil {
							return err, 0, 0
						}
					}
					v_.Timestep = int(timestep)
				}
			}

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package metricdispatch provides a unified interface for loading and caching job metric data.
//
// This package serves as a central dispatcher that routes metric data requests to the appropriate
// backend based on job state. For running jobs, data is fetched from the metric store (e.g., cc-metric-store).
// For completed jobs, data is retrieved from the file-based job archive.
//
// # Key Features
//
//   - Automatic backend selection based on job state (running vs. archived)
//   - LRU cache for performance optimization (128 MB default cache size)
//   - Data resampling using Largest Triangle Three Bucket algorithm for archived data
//   - Automatic statistics series generation for jobs with many nodes
//   - Support for scoped metrics (node, socket, accelerator, core)
//
// # Cache Behavior
//
// Cached data has different TTL (time-to-live) values depending on job state:
//   - Running jobs: 2 minutes (data changes frequently)
//   - Completed jobs: 5 hours (data is static)
//
// The cache key is based on job ID, state, requested metrics, scopes, and resolution.
//
// # Usage
//
// The primary entry point is LoadData, which automatically handles both running and archived jobs:
//
//	jobData, err := metricdispatch.LoadData(job, metrics, scopes, ctx, resolution)
//	if err != nil {
//	    // Handle error
//	}
//
// For statistics only, use LoadJobStats, LoadScopedJobStats, or LoadAverages depending on the required format.
package metricdispatch

import (
	"context"
	"fmt"
	"math"
	"time"

	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/lrucache"
	"github.com/ClusterCockpit/cc-lib/v2/resampler"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

// cache is an LRU cache with 128 MB capacity for storing loaded job metric data.
// The cache reduces load on both the metric store and archive backends.
var cache *lrucache.Cache = lrucache.New(128 * 1024 * 1024)

// cacheKey generates a unique cache key for a job's metric data based on job ID, state,
// requested metrics, scopes, and resolution. Duration and StartTime are intentionally excluded
// because job.ID is more unique and the cache TTL ensures entries don't persist indefinitely.
func cacheKey(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	resampleAlgo string,
) string {
	return fmt.Sprintf("%d(%s):[%v],[%v]-%d-%s",
		*job.ID, job.State, metrics, scopes, resolution, resampleAlgo)
}

// LoadData retrieves metric data for a job from the appropriate backend (memory store for running jobs,
// archive for completed jobs) and applies caching, resampling, and statistics generation as needed.
//
// For running jobs or when archive is disabled, data is fetched from the metric store.
// For completed archived jobs, data is loaded from the job archive and resampled if needed.
//
// Parameters:
//   - job: The job for which to load metric data
//   - metrics: List of metric names to load (nil loads all metrics for the cluster)
//   - scopes: Metric scopes to include (nil defaults to node scope)
//   - ctx: Context for cancellation and timeouts
//   - resolution: Target number of data points for resampling (only applies to archived data)
//
// Returns the loaded job data and any error encountered. For partial errors (some metrics failed),
// the function returns the successfully loaded data with a warning logged.
func LoadData(job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
	resolution int,
	resampleAlgo string,
) (schema.JobData, error) {
	data := cache.Get(cacheKey(job, metrics, scopes, resolution, resampleAlgo), func() (_ any, ttl time.Duration, size int) {
		var jd schema.JobData
		var err error

		if job.State == schema.JobStateRunning ||
			job.MonitoringStatus == schema.MonitoringStatusRunningOrArchiving {

			ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
			if err != nil {
				cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
					job.Cluster, job.SubCluster, err.Error())
				return err, 0, 0
			}

			if scopes == nil {
				scopes = append(scopes, schema.MetricScopeNode)
			}

			if metrics == nil {
				cluster := archive.GetCluster(job.Cluster)
				for _, mc := range cluster.MetricConfig {
					metrics = append(metrics, mc.Name)
				}
			}

			jd, err = ms.LoadData(job, metrics, scopes, ctx, resolution)
			if err != nil {
				if len(jd) != 0 {
					cclog.Warnf("partial error loading metrics from store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
						job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
				} else {
					cclog.Warnf("failed to load job data from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
						job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
					return err, 0, 0
				}
			}
			size = jd.Size()
		} else {
			var jdTemp schema.JobData
			jdTemp, err = archive.GetHandle().LoadJobData(job)
			if err != nil {
				cclog.Warnf("failed to load job data from archive for job %d (user: %s, project: %s, cluster: %s-%s): %s",
					job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
				return err, 0, 0
			}

			jd = deepCopy(jdTemp)

			// Resample archived data using Largest Triangle Three Bucket algorithm to reduce data points
			// to the requested resolution, improving transfer performance and client-side rendering.
			for _, v := range jd {
				for _, v_ := range v {
					timestep := int64(0)
					for i := 0; i < len(v_.Series); i += 1 {
						v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))
						if err != nil {
							return err, 0, 0
						}
					}
					v_.Timestep = int(timestep)
				}
			}

			// Filter job data to only include requested metrics and scopes, avoiding unnecessary data transfer.
			if metrics != nil || scopes != nil {
				if metrics == nil {
					metrics = make([]string, 0, len(jd))
					for k := range jd {
						metrics = append(metrics, k)
					}
				}

				res := schema.JobData{}
				for _, metric := range metrics {
					if perscope, ok := jd[metric]; ok {
						if len(perscope) > 1 {
							subset := make(map[schema.MetricScope]*schema.JobMetric)
							for _, scope := range scopes {
								if jm, ok := perscope[scope]; ok {
									subset[scope] = jm
								}
							}

							if len(subset) > 0 {
								perscope = subset
							}
						}

						res[metric] = perscope
					}
				}
				jd = res
			}
			size = jd.Size()
		}

		ttl = 5 * time.Hour
		if job.State == schema.JobStateRunning {
			ttl = 2 * time.Minute
		}

		// Generate statistics series for jobs with many nodes to enable min/median/max graphs
		// instead of overwhelming the UI with individual node lines. Note that newly calculated
		// statistics use min/median/max, while archived statistics may use min/mean/max.
		const maxSeriesSize int = 8
		for _, scopes := range jd {
			for _, jm := range scopes {
				if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
					continue
				}

				jm.AddStatisticsSeries()
			}
		}

		nodeScopeRequested := false
		for _, scope := range scopes {
			if scope == schema.MetricScopeNode {
				nodeScopeRequested = true
			}
		}

		if nodeScopeRequested {
			jd.AddNodeScope("flops_any")
			jd.AddNodeScope("mem_bw")
		}

		// Round Resulting Stat Values
		jd.RoundMetricStats()

		return jd, ttl, size
	})

	if err, ok := data.(error); ok {
		cclog.Errorf("error in cached dataset for job %d: %s", job.JobID, err.Error())
		return nil, err
	}

	return data.(schema.JobData), nil
}

// LoadAverages computes average values for the specified metrics across all nodes of a job.
// For running jobs, it loads statistics from the metric store. For completed jobs, it uses
// the pre-calculated averages from the job archive. The results are appended to the data slice.
func LoadAverages(
	job *schema.Job,
	metrics []string,
	data [][]schema.Float,
	ctx context.Context,
) error {
	if job.State != schema.JobStateRunning {
		return archive.LoadAveragesFromArchive(job, metrics, data) // #166 change also here?
	}

	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			job.Cluster, job.SubCluster, err.Error())
		return err
	}

	stats, err := ms.LoadStats(job, metrics, ctx)
	if err != nil {
		cclog.Warnf("failed to load statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
		return err
	}

	for i, m := range metrics {
		nodes, ok := stats[m]
		if !ok {
			data[i] = append(data[i], schema.NaN)
			continue
		}

		sum := 0.0
		for _, node := range nodes {
			sum += node.Avg
		}
		data[i] = append(data[i], schema.Float(sum))
	}

	return nil
}

// LoadScopedJobStats retrieves job statistics organized by metric scope (node, socket, core, accelerator).
// For running jobs, statistics are computed from the metric store. For completed jobs, pre-calculated
// statistics are loaded from the job archive.
func LoadScopedJobStats(
	job *schema.Job,
	metrics []string,
	scopes []schema.MetricScope,
	ctx context.Context,
) (schema.ScopedJobStats, error) {
	if job.State != schema.JobStateRunning {
		return archive.LoadScopedStatsFromArchive(job, metrics, scopes)
	}

	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			job.Cluster, job.SubCluster, err.Error())
		return nil, err
	}

	scopedStats, err := ms.LoadScopedStats(job, metrics, scopes, ctx)
	if err != nil {
		cclog.Warnf("failed to load scoped statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
		return nil, err
	}

	// Round Resulting Stat Values
	scopedStats.RoundScopedMetricStats()

	return scopedStats, nil
}

// LoadJobStats retrieves aggregated statistics (min/avg/max) for each requested metric across all job nodes.
// For running jobs, statistics are computed from the metric store. For completed jobs, pre-calculated
// statistics are loaded from the job archive.
func LoadJobStats(
	job *schema.Job,
	metrics []string,
	ctx context.Context,
) (map[string]schema.MetricStatistics, error) {
	if job.State != schema.JobStateRunning {
		return archive.LoadStatsFromArchive(job, metrics)
	}

	ms, err := GetMetricDataRepo(job.Cluster, job.SubCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			job.Cluster, job.SubCluster, err.Error())
		return nil, err
	}

	data := make(map[string]schema.MetricStatistics, len(metrics))

	stats, err := ms.LoadStats(job, metrics, ctx)
	if err != nil {
		cclog.Warnf("failed to load statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
		return data, err
	}

	for _, m := range metrics {
		sum, avg, min, max := 0.0, 0.0, 0.0, 0.0
		nodes, ok := stats[m]
		if !ok {
			data[m] = schema.MetricStatistics{Min: min, Avg: avg, Max: max}
			continue
		}

		for _, node := range nodes {
			sum += node.Avg
			min = math.Min(min, node.Min)
			max = math.Max(max, node.Max)
		}

		data[m] = schema.MetricStatistics{
			Avg: (math.Round((sum/float64(job.NumNodes))*100) / 100),
			Min: (math.Round(min*100) / 100),
			Max: (math.Round(max*100) / 100),
		}
	}

	return data, nil
}

// LoadNodeData retrieves metric data for specific nodes in a cluster within a time range.
// This is used for node monitoring views and system status pages. Data is always fetched from
// the metric store (not the archive) since it's for current/recent node status monitoring.
//
// Returns a nested map structure: node -> metric -> scoped data.
// FIXME: Add support for subcluster specific cc-metric-stores
func LoadNodeData(
	cluster string,
	metrics, nodes []string,
	scopes []schema.MetricScope,
	from, to time.Time,
	ctx context.Context,
) (map[string]map[string][]*schema.JobMetric, error) {
	if metrics == nil {
		for _, m := range archive.GetCluster(cluster).MetricConfig {
			metrics = append(metrics, m.Name)
		}
	}

	ms, err := GetMetricDataRepo(cluster, "")
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s: %s",
			cluster, err.Error())
		return nil, err
	}

	data, err := ms.LoadNodeData(cluster, metrics, nodes, scopes, from, to, ctx)
	if err != nil {
		if len(data) != 0 {
			cclog.Warnf("partial error loading node data from metric store for cluster %s: %s", cluster, err.Error())
		} else {
			cclog.Warnf("failed to load node data from metric store for cluster %s: %s", cluster, err.Error())
			return nil, err
		}
	}

	if data == nil {
		return nil, fmt.Errorf("metric store for cluster '%s' does not support node data queries", cluster)
	}

	return data, nil
}

// LoadNodeListData retrieves time-series metric data for multiple nodes within a time range,
// with optional resampling and automatic statistics generation for large datasets.
// This is used for comparing multiple nodes or displaying node status over time.
//
// Returns a map of node names to their job-like metric data structures.
func LoadNodeListData(
	cluster, subCluster string,
	nodes []string,
	metrics []string,
	scopes []schema.MetricScope,
	resolution int,
	from, to time.Time,
	ctx context.Context,
) (map[string]schema.JobData, error) {
	if metrics == nil {
		for _, m := range archive.GetCluster(cluster).MetricConfig {
			metrics = append(metrics, m.Name)
		}
	}

	ms, err := GetMetricDataRepo(cluster, subCluster)
	if err != nil {
		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
			cluster, subCluster, err.Error())
		return nil, err
	}

	data, err := ms.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, resolution, from, to, ctx)
	if err != nil {
		if len(data) != 0 {
			cclog.Warnf("partial error loading node list data from metric store for cluster %s, subcluster %s: %s",
				cluster, subCluster, err.Error())
		} else {
			cclog.Warnf("failed to load node list data from metric store for cluster %s, subcluster %s: %s",
				cluster, subCluster, err.Error())
			return nil, err
		}
	}

	// Generate statistics series for datasets with many series to improve visualization performance.
	// Statistics are calculated as min/median/max.
	const maxSeriesSize int = 8
	for _, jd := range data {
		for _, scopes := range jd {
			for _, jm := range scopes {
				if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
					continue
				}
				jm.AddStatisticsSeries()
			}
		}
	}

	if data == nil {
		return nil, fmt.Errorf("metric store for cluster '%s' does not support node list queries", cluster)
	}

	return data, nil
}

// deepCopy creates a deep copy of JobData to prevent cache corruption when modifying
// archived data (e.g., during resampling). This ensures the cached archive data remains
// immutable while allowing per-request transformations.
func deepCopy(source schema.JobData) schema.JobData {
	result := make(schema.JobData, len(source))

	for metricName, scopeMap := range source {
		result[metricName] = make(map[schema.MetricScope]*schema.JobMetric, len(scopeMap))

		for scope, jobMetric := range scopeMap {
			result[metricName][scope] = copyJobMetric(jobMetric)
		}
	}

	return result
}

func copyJobMetric(src *schema.JobMetric) *schema.JobMetric {
	dst := &schema.JobMetric{
		Timestep: src.Timestep,
		Unit:     src.Unit,
		Series:   make([]schema.Series, len(src.Series)),
	}

	for i := range src.Series {
		dst.Series[i] = copySeries(&src.Series[i])
	}

	if src.StatisticsSeries != nil {
		dst.StatisticsSeries = copyStatisticsSeries(src.StatisticsSeries)
	}

	return dst
}

func copySeries(src *schema.Series) schema.Series {
	dst := schema.Series{
		Hostname:   src.Hostname,
		ID:         src.ID,
		Statistics: src.Statistics,
		Data:       make([]schema.Float, len(src.Data)),
	}

	copy(dst.Data, src.Data)
	return dst
}

func copyStatisticsSeries(src *schema.StatsSeries) *schema.StatsSeries {
	dst := &schema.StatsSeries{
		Min:    make([]schema.Float, len(src.Min)),
		Mean:   make([]schema.Float, len(src.Mean)),
		Median: make([]schema.Float, len(src.Median)),
		Max:    make([]schema.Float, len(src.Max)),
	}

	copy(dst.Min, src.Min)
	copy(dst.Mean, src.Mean)
	copy(dst.Median, src.Median)
	copy(dst.Max, src.Max)

	if len(src.Percentiles) > 0 {
		dst.Percentiles = make(map[int][]schema.Float, len(src.Percentiles))
		for percentile, values := range src.Percentiles {
			dst.Percentiles[percentile] = make([]schema.Float, len(values))
			copy(dst.Percentiles[percentile], values)
		}
	}

	return dst
}

-      // Resample archived data using Largest Triangle Three Bucket algorithm to reduce data points

-      // to the requested resolution, improving transfer performance and client-side rendering.

+      // Resample archived data to reduce data points to the requested resolution,

+      // improving transfer performance and client-side rendering.

+      resampleFn, rfErr := resampler.GetResampler(resampleAlgo)

-            v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))

+            v_.Series[i].Data, timestep, err = resampleFn(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))


There are other callers of `LoadData` that need updating. Let me find and fix all of them:


internal/metricdispatch/dataLoader.go:32://	jobData, err := metricdispatch.LoadData(job, metrics, scopes, ctx, resolution)
internal/api/api_test.go:358:		data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60)
internal/api/job.go:304:		data, err = metricdispatch.LoadData(job, nil, scopes, r.Context(), resolution)
internal/api/job.go:400:	data, err := metricdispatch.LoadData(job, metrics, scopes, r.Context(), resolution)
internal/archiver/README.md:109:jobData, err := metricdispatch.LoadData(job, allMetrics, scopes, ctx, 300)
internal/archiver/archiver.go:62:	jobData, err := metricdispatch.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s)
internal/graph/schema.resolvers.go:523:	data, err := metricdispatch.LoadData(job, metrics, scopes, ctx, *resolution, algoName)
internal/graph/util.go:58:		jobdata, err := metricdispatch.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0)


I need to fix the other callers - they should pass empty string (default LTTB):


   355→	}
   356→
   357→	t.Run("CheckArchive", func(t *testing.T) {
   358→		data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60)
   359→		if err != nil {

	}

	t.Run("CheckArchive", func(t *testing.T) {
		data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60)
		if err != nil {


   300→		resolution = max(resolution, mc.Timestep)
   301→	}
   302→
   303→	if r.URL.Query().Get("all-metrics") == "true" {
   304→		data, err = metricdispatch.LoadData(job, nil, scopes, r.Context(), resolution)
   305→		if err != nil {
   306→			cclog.Warnf("REST: error while loading all-metrics job data for JobID %d on %s", job.JobID, job.Cluster)
   307→			return

		resolution = max(resolution, mc.Timestep)
	}

	if r.URL.Query().Get("all-metrics") == "true" {
		data, err = metricdispatch.LoadData(job, nil, scopes, r.Context(), resolution)
		if err != nil {
			cclog.Warnf("REST: error while loading all-metrics job data for JobID %d on %s", job.JobID, job.Cluster)
			return


   396→	for _, mc := range metricConfigs {
   397→		resolution = max(resolution, mc.Timestep)
   398→	}
   399→
   400→	data, err := metricdispatch.LoadData(job, metrics, scopes, r.Context(), resolution)
   401→	if err != nil {
   402→		cclog.Warnf("REST: error while loading job data for JobID %d on %s", job.JobID, job.Cluster)
   403→		return

	for _, mc := range metricConfigs {
		resolution = max(resolution, mc.Timestep)
	}

	data, err := metricdispatch.LoadData(job, metrics, scopes, r.Context(), resolution)
	if err != nil {
		cclog.Warnf("REST: error while loading job data for JobID %d on %s", job.JobID, job.Cluster)
		return


    59→		scopes = append(scopes, schema.MetricScopeAccelerator)
    60→	}
    61→
    62→	jobData, err := metricdispatch.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s)
    63→	if err != nil {

		scopes = append(scopes, schema.MetricScopeAccelerator)
	}

	jobData, err := metricdispatch.LoadData(job, allMetrics, scopes, ctx, 0) // 0 Resulotion-Value retrieves highest res (60s)
	if err != nil {


    55→		// 	resolution = max(resolution, mc.Timestep)
    56→		// }
    57→
    58→		jobdata, err := metricdispatch.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0)
    59→		if err != nil {

		// 	resolution = max(resolution, mc.Timestep)
		// }

		jobdata, err := metricdispatch.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0)
		if err != nil {


data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60)

data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60, "")


The file /Users/jan/tmp/cc-backend/internal/api/api_test.go has been updated successfully.

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package api_test

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/api"
	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/auth"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/graph"
	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
	"github.com/ClusterCockpit/cc-backend/internal/repository"
	"github.com/ClusterCockpit/cc-backend/pkg/archive"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	"github.com/go-chi/chi/v5"

	_ "github.com/mattn/go-sqlite3"
)

func setup(t *testing.T) *api.RestAPI {
	repository.ResetConnection()

	const testconfig = `{
	"main": {
	   "addr":            "0.0.0.0:8080",
	   "validate": false,
     "api-allowed-ips": [
       "*"
      ]
  },
  "metric-store": {
    "checkpoints": {
      "interval": "12h"
    },
    "retention-in-memory": "48h",
    "memory-cap": 100
  },
	"archive": {
		 "kind": "file",
		 "path": "./var/job-archive"
	},
	"auth": {
     "jwts": {
     "max-age": "2m"
  }
  }
}`
	const testclusterJSON = `{
        "name": "testcluster",
		"subClusters": [
			{
				"name": "sc1",
				"nodes": "host123,host124,host125",
				"processorType": "Intel Core i7-4770",
				"socketsPerNode": 1,
				"coresPerSocket": 4,
				"threadsPerCore": 2,
                "flopRateScalar": {
                  "unit": {
                    "prefix": "G",
                    "base": "F/s"
                  },
                  "value": 14
                },
                "flopRateSimd": {
                  "unit": {
                    "prefix": "G",
                    "base": "F/s"
                  },
                  "value": 112
                },
                "memoryBandwidth": {
                  "unit": {
                    "prefix": "G",
                    "base": "B/s"
                  },
                  "value": 24
                },
                "numberOfNodes": 70,
				"topology": {
					"node": [0, 1, 2, 3, 4, 5, 6, 7],
					"socket": [[0, 1, 2, 3, 4, 5, 6, 7]],
					"memoryDomain": [[0, 1, 2, 3, 4, 5, 6, 7]],
					"die": [[0, 1, 2, 3, 4, 5, 6, 7]],
					"core": [[0], [1], [2], [3], [4], [5], [6], [7]]
				}
			}
		],
		"metricConfig": [
			{
				"name": "load_one",
			    "unit": { "base": ""},
				"scope": "node",
				"timestep": 60,
                "aggregation": "avg",
				"peak": 8,
				"normal": 0,
				"caution": 0,
				"alert": 0
			}
		]
	}`

	cclog.Init("info", true)
	tmpdir := t.TempDir()
	jobarchive := filepath.Join(tmpdir, "job-archive")
	if err := os.Mkdir(jobarchive, 0o777); err != nil {
		t.Fatal(err)
	}

	if err := os.WriteFile(filepath.Join(jobarchive, "version.txt"), fmt.Appendf(nil, "%d", 3), 0o666); err != nil {
		t.Fatal(err)
	}

	if err := os.Mkdir(filepath.Join(jobarchive, "testcluster"), 0o777); err != nil {
		t.Fatal(err)
	}

	if err := os.WriteFile(filepath.Join(jobarchive, "testcluster", "cluster.json"), []byte(testclusterJSON), 0o666); err != nil {
		t.Fatal(err)
	}

	dbfilepath := filepath.Join(tmpdir, "test.db")
	err := repository.MigrateDB(dbfilepath)
	if err != nil {
		t.Fatal(err)
	}

	cfgFilePath := filepath.Join(tmpdir, "config.json")
	if err := os.WriteFile(cfgFilePath, []byte(testconfig), 0o666); err != nil {
		t.Fatal(err)
	}

	ccconf.Init(cfgFilePath)
	metricstore.MetricStoreHandle = &metricstore.InternalMetricStore{}

	// Load and check main configuration
	if cfg := ccconf.GetPackageConfig("main"); cfg != nil {
		config.Init(cfg)
	} else {
		cclog.Abort("Main configuration must be present")
	}
	archiveCfg := fmt.Sprintf("{\"kind\": \"file\",\"path\": \"%s\"}", jobarchive)

	repository.Connect(dbfilepath)

	if err := archive.Init(json.RawMessage(archiveCfg)); err != nil {
		t.Fatal(err)
	}

	// metricstore initialization removed - it's initialized via callback in tests

	archiver.Start(repository.GetJobRepository(), context.Background())

	if cfg := ccconf.GetPackageConfig("auth"); cfg != nil {
		auth.Init(&cfg)
	} else {
		cclog.Warn("Authentication disabled due to missing configuration")
		auth.Init(nil)
	}

	graph.Init()

	return api.New()
}

func cleanup() {
	if err := archiver.Shutdown(5 * time.Second); err != nil {
		cclog.Warnf("Archiver shutdown timeout in tests: %v", err)
	}
}

/*
* This function starts a job, stops it, and then reads its data from the job-archive.
* Do not run sub-tests in parallel! Tests should not be run in parallel at all, because
* at least `setup` modifies global state.
 */
func TestRestApi(t *testing.T) {
	restapi := setup(t)
	t.Cleanup(cleanup)
	testData := schema.JobData{
		"load_one": map[schema.MetricScope]*schema.JobMetric{
			schema.MetricScopeNode: {
				Unit:     schema.Unit{Base: "load"},
				Timestep: 60,
				Series: []schema.Series{
					{
						Hostname:   "host123",
						Statistics: schema.MetricStatistics{Min: 0.1, Avg: 0.2, Max: 0.3},
						Data:       []schema.Float{0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.3, 0.3, 0.3},
					},
				},
			},
		},
	}

	metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
		return testData, nil
	}

	r := chi.NewRouter()
	restapi.MountAPIRoutes(r)

	var TestJobID int64 = 123
	TestClusterName := "testcluster"
	var TestStartTime int64 = 123456789

	const startJobBody string = `{
    "jobId":            123,
		"user":             "testuser",
		"project":          "testproj",
		"cluster":          "testcluster",
		"partition":        "default",
		"walltime":         3600,
		"arrayJobId":       0,
		"numNodes":         1,
		"numHwthreads":     8,
		"numAcc":           0,
		"shared":           "none",
		"monitoringStatus": 1,
		"smt":              1,
		"resources": [
			{
				"hostname": "host123",
				"hwthreads": [0, 1, 2, 3, 4, 5, 6, 7]
			}
		],
		"metaData":  { "jobScript": "blablabla..." },
		"startTime": 123456789
	}`

	const contextUserKey repository.ContextKey = "user"
	contextUserValue := &schema.User{
		Username:   "testuser",
		Projects:   make([]string, 0),
		Roles:      []string{"user"},
		AuthType:   0,
		AuthSource: 2,
	}

	if ok := t.Run("StartJob", func(t *testing.T) {
		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(startJobBody)))
		recorder := httptest.NewRecorder()

		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)

		r.ServeHTTP(recorder, req.WithContext(ctx))
		response := recorder.Result()
		if response.StatusCode != http.StatusCreated {
			t.Fatal(response.Status, recorder.Body.String())
		}
		// resolver := graph.GetResolverInstance()
		restapi.JobRepository.SyncJobs()
		job, err := restapi.JobRepository.Find(&TestJobID, &TestClusterName, &TestStartTime)
		if err != nil {
			t.Fatal(err)
		}

		// job.Tags, err = resolver.Job().Tags(ctx, job)
		// if err != nil {
		// 	t.Fatal(err)
		// }

		if job.JobID != 123 ||
			job.User != "testuser" ||
			job.Project != "testproj" ||
			job.Cluster != "testcluster" ||
			job.SubCluster != "sc1" ||
			job.Partition != "default" ||
			job.Walltime != 3600 ||
			job.ArrayJobID != 0 ||
			job.NumNodes != 1 ||
			job.NumHWThreads != 8 ||
			job.NumAcc != 0 ||
			job.MonitoringStatus != 1 ||
			job.SMT != 1 ||
			!reflect.DeepEqual(job.Resources, []*schema.Resource{{Hostname: "host123", HWThreads: []int{0, 1, 2, 3, 4, 5, 6, 7}}}) ||
			job.StartTime != 123456789 {
			t.Fatalf("unexpected job properties: %#v", job)
		}

		// if len(job.Tags) != 1 || job.Tags[0].Type != "testTagType" || job.Tags[0].Name != "testTagName" || job.Tags[0].Scope != "testuser" {
		// 	t.Fatalf("unexpected tags: %#v", job.Tags)
		// }
	}); !ok {
		return
	}

	const stopJobBody string = `{
        "jobId":     123,
		"startTime": 123456789,
		"cluster":   "testcluster",

		"jobState": "completed",
		"stopTime": 123457789
	}`

	var stoppedJob *schema.Job
	if ok := t.Run("StopJob", func(t *testing.T) {
		req := httptest.NewRequest(http.MethodPost, "/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBody)))
		recorder := httptest.NewRecorder()

		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)

		r.ServeHTTP(recorder, req.WithContext(ctx))
		response := recorder.Result()
		if response.StatusCode != http.StatusOK {
			t.Fatal(response.Status, recorder.Body.String())
		}

		// Archiving happens asynchronously, will be completed in cleanup
		job, err := restapi.JobRepository.Find(&TestJobID, &TestClusterName, &TestStartTime)
		if err != nil {
			t.Fatal(err)
		}

		if job.State != schema.JobStateCompleted {
			t.Fatal("expected job to be completed")
		}

		if job.Duration != (123457789 - 123456789) {
			t.Fatalf("unexpected job properties: %#v", job)
		}

		job.MetaData, err = restapi.JobRepository.FetchMetadata(job)
		if err != nil {
			t.Fatal(err)
		}

		if !reflect.DeepEqual(job.MetaData, map[string]string{"jobScript": "blablabla..."}) {
			t.Fatalf("unexpected job.metaData: %#v", job.MetaData)
		}

		stoppedJob = job
	}); !ok {
		return
	}

	t.Run("CheckArchive", func(t *testing.T) {
		data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60)
		if err != nil {
			t.Fatal(err)
		}

		if !reflect.DeepEqual(data, testData) {
			t.Fatal("unexpected data fetched from archive")
		}
	})

	t.Run("CheckDoubleStart", func(t *testing.T) {
		// Starting a job with the same jobId and cluster should only be allowed if the startTime is far appart!
		body := strings.ReplaceAll(startJobBody, `"startTime": 123456789`, `"startTime": 123456790`)

		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(body)))
		recorder := httptest.NewRecorder()

		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)

		r.ServeHTTP(recorder, req.WithContext(ctx))
		response := recorder.Result()
		if response.StatusCode != http.StatusUnprocessableEntity {
			t.Fatal(response.Status, recorder.Body.String())
		}
	})

	const startJobBodyFailed string = `{
        "jobId":            12345,
		"user":             "testuser",
		"project":          "testproj",
		"cluster":          "testcluster",
		"partition":        "default",
		"walltime":         3600,
		"numNodes":         1,
		"shared":        	"none",
		"monitoringStatus": 1,
		"smt":              1,
		"resources": [
			{
				"hostname": "host123"
			}
		],
		"startTime": 12345678
	}`

	ok := t.Run("StartJobFailed", func(t *testing.T) {
		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(startJobBodyFailed)))
		recorder := httptest.NewRecorder()

		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)

		r.ServeHTTP(recorder, req.WithContext(ctx))
		response := recorder.Result()
		if response.StatusCode != http.StatusCreated {
			t.Fatal(response.Status, recorder.Body.String())
		}
	})
	if !ok {
		t.Fatal("subtest failed")
	}

	time.Sleep(1 * time.Second)
	restapi.JobRepository.SyncJobs()

	const stopJobBodyFailed string = `{
    "jobId":     12345,
		"cluster":   "testcluster",

		"jobState": "failed",
		"stopTime": 12355678
	}`

	ok = t.Run("StopJobFailed", func(t *testing.T) {
		req := httptest.NewRequest(http.MethodPost, "/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBodyFailed)))
		recorder := httptest.NewRecorder()

		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)

		r.ServeHTTP(recorder, req.WithContext(ctx))
		response := recorder.Result()
		if response.StatusCode != http.StatusOK {
			t.Fatal(response.Status, recorder.Body.String())
		}

		// Archiving happens asynchronously, will be completed in cleanup
		jobid, cluster := int64(12345), "testcluster"
		job, err := restapi.JobRepository.Find(&jobid, &cluster, nil)
		if err != nil {
			t.Fatal(err)
		}

		if job.State != schema.JobStateFailed {
			t.Fatal("expected job to be failed")
		}
	})
	if !ok {
		t.Fatal("subtest failed")
	}

	t.Run("GetUsedNodesNoRunning", func(t *testing.T) {
		contextUserValue := &schema.User{
			Username:   "testuser",
			Projects:   make([]string, 0),
			Roles:      []string{"api"},
			AuthType:   0,
			AuthSource: 2,
		}

		req := httptest.NewRequest(http.MethodGet, "/jobs/used_nodes?ts=123456790", nil)
		recorder := httptest.NewRecorder()

		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)

		r.ServeHTTP(recorder, req.WithContext(ctx))
		response := recorder.Result()
		if response.StatusCode != http.StatusOK {
			t.Fatal(response.Status, recorder.Body.String())
		}

		var result api.GetUsedNodesAPIResponse
		if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
			t.Fatal(err)
		}

		if result.UsedNodes == nil {
			t.Fatal("expected usedNodes to be non-nil")
		}

		if len(result.UsedNodes) != 0 {
			t.Fatalf("expected no used nodes for stopped jobs, got: %v", result.UsedNodes)
		}
	})
}

// TestStopJobWithReusedJobId verifies that stopping a recently started job works
// even when an older job with the same jobId exists in the job table (e.g. with
// state "failed"). This is a regression test for the bug where Find() on the job
// table would match the old job instead of the new one still in job_cache.
func TestStopJobWithReusedJobId(t *testing.T) {
	restapi := setup(t)
	t.Cleanup(cleanup)

	testData := schema.JobData{
		"load_one": map[schema.MetricScope]*schema.JobMetric{
			schema.MetricScopeNode: {
				Unit:     schema.Unit{Base: "load"},
				Timestep: 60,
				Series: []schema.Series{
					{
						Hostname:   "host123",
						Statistics: schema.MetricStatistics{Min: 0.1, Avg: 0.2, Max: 0.3},
						Data:       []schema.Float{0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.3, 0.3, 0.3},
					},
				},
			},
		},
	}

	metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
		return testData, nil
	}

	r := chi.NewRouter()
	restapi.MountAPIRoutes(r)

	const contextUserKey repository.ContextKey = "user"
	contextUserValue := &schema.User{
		Username:   "testuser",
		Projects:   make([]string, 0),
		Roles:      []string{"user"},
		AuthType:   0,
		AuthSource: 2,
	}

	// Step 1: Start the first job (jobId=999)
	const startJobBody1 string = `{
		"jobId":            999,
		"user":             "testuser",
		"project":          "testproj",
		"cluster":          "testcluster",
		"partition":        "default",
		"walltime":         3600,
		"numNodes":         1,
		"numHwthreads":     8,
		"numAcc":           0,
		"shared":           "none",
		"monitoringStatus": 1,
		"smt":              1,
		"resources": [{"hostname": "host123", "hwthreads": [0, 1, 2, 3, 4, 5, 6, 7]}],
		"startTime":        200000000
	}`

	if ok := t.Run("StartFirstJob", func(t *testing.T) {
		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(startJobBody1)))
		recorder := httptest.NewRecorder()
		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
		r.ServeHTTP(recorder, req.WithContext(ctx))
		if recorder.Result().StatusCode != http.StatusCreated {
			t.Fatal(recorder.Result().Status, recorder.Body.String())
		}
	}); !ok {
		return
	}

	// Step 2: Sync to move job from cache to job table, then stop it as "failed"
	time.Sleep(1 * time.Second)
	restapi.JobRepository.SyncJobs()

	const stopJobBody1 string = `{
		"jobId":     999,
		"startTime": 200000000,
		"cluster":   "testcluster",
		"jobState":  "failed",
		"stopTime":  200001000
	}`

	if ok := t.Run("StopFirstJobAsFailed", func(t *testing.T) {
		req := httptest.NewRequest(http.MethodPost, "/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBody1)))
		recorder := httptest.NewRecorder()
		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
		r.ServeHTTP(recorder, req.WithContext(ctx))
		if recorder.Result().StatusCode != http.StatusOK {
			t.Fatal(recorder.Result().Status, recorder.Body.String())
		}

		jobid, cluster := int64(999), "testcluster"
		job, err := restapi.JobRepository.Find(&jobid, &cluster, nil)
		if err != nil {
			t.Fatal(err)
		}
		if job.State != schema.JobStateFailed {
			t.Fatalf("expected first job to be failed, got: %s", job.State)
		}
	}); !ok {
		return
	}

	// Wait for archiving to complete
	time.Sleep(1 * time.Second)

	// Step 3: Start a NEW job with the same jobId=999 but different startTime.
	// This job will sit in job_cache (not yet synced).
	const startJobBody2 string = `{
		"jobId":            999,
		"user":             "testuser",
		"project":          "testproj",
		"cluster":          "testcluster",
		"partition":        "default",
		"walltime":         3600,
		"numNodes":         1,
		"numHwthreads":     8,
		"numAcc":           0,
		"shared":           "none",
		"monitoringStatus": 1,
		"smt":              1,
		"resources": [{"hostname": "host123", "hwthreads": [0, 1, 2, 3, 4, 5, 6, 7]}],
		"startTime":        300000000
	}`

	if ok := t.Run("StartSecondJob", func(t *testing.T) {
		req := httptest.NewRequest(http.MethodPost, "/jobs/start_job/", bytes.NewBuffer([]byte(startJobBody2)))
		recorder := httptest.NewRecorder()
		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
		r.ServeHTTP(recorder, req.WithContext(ctx))
		if recorder.Result().StatusCode != http.StatusCreated {
			t.Fatal(recorder.Result().Status, recorder.Body.String())
		}
	}); !ok {
		return
	}

	// Step 4: Stop the second job WITHOUT syncing first.
	// Before the fix, this would fail because Find() on the job table would
	// match the old failed job (jobId=999) and reject with "already stopped".
	const stopJobBody2 string = `{
		"jobId":     999,
		"startTime": 300000000,
		"cluster":   "testcluster",
		"jobState":  "completed",
		"stopTime":  300001000
	}`

	t.Run("StopSecondJobBeforeSync", func(t *testing.T) {
		req := httptest.NewRequest(http.MethodPost, "/jobs/stop_job/", bytes.NewBuffer([]byte(stopJobBody2)))
		recorder := httptest.NewRecorder()
		ctx := context.WithValue(req.Context(), contextUserKey, contextUserValue)
		r.ServeHTTP(recorder, req.WithContext(ctx))
		if recorder.Result().StatusCode != http.StatusOK {
			t.Fatalf("expected stop to succeed for cached job, got: %s %s",
				recorder.Result().Status, recorder.Body.String())
		}
	})
}

-    data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60)

+    data, err := metricdispatch.LoadData(stoppedJob, []string{"load_one"}, []schema.MetricScope{schema.MetricScopeNode}, context.Background(), 60, "")


data, err = metricdispatch.LoadData(job, nil, scopes, r.Context(), resolution)

data, err = metricdispatch.LoadData(job, nil, scopes, r.Context(), resolution, "")