From 0724b2dc60ffebbb7198e49ee36de3c964f3f324 Mon Sep 17 00:00:00 2001
From: Jan Eitzinger <jan@moebiusband.org>
Date: Wed, 1 Jul 2026 19:14:12 +0200
Subject: [PATCH] Adopt cc-lib hierarchical JobData/Statistics structs

cc-lib changed JobData, ScopedJobStats and Job.Statistics from flat maps
to structs (a .Metrics map plus array-valued Groups) to represent
filesystem (and future interconnect) metric groups. Migrate all
construction, indexing and iteration to the new API across the archive
backends, metricstore query path, metric dispatcher, archiver, importer,
tagger, taskmanager, repository and API layers.

Semantics: DecodeJobStats now projects JobData.Groups into
ScopedJobStats.Groups, and the archiver derives per-filesystem node-scope
statistics into Job.Statistics.Groups. deepCopy, resampling and the
metric/scope filter are group-aware. The metricstore internal storage
(buffers, selector tree, checkpoint/parquet) is unchanged; all conversion
stays at the LoadData/archive-codec seam.

Note: requires the corresponding cc-lib release; bump the cc-lib
dependency version once tagged (a local go.mod replace was used during
development and is intentionally not committed).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 internal/api/api_test.go                      |  8 +-
 internal/api/job.go                           |  2 +-
 internal/api/nats_test.go                     |  4 +-
 internal/archiver/archiver.go                 | 46 +++++++++-
 internal/graph/schema.resolvers.go            |  6 +-
 internal/graph/util.go                        |  2 +-
 internal/importer/initDB.go                   |  2 +-
 internal/metricdispatch/dataLoader.go         | 83 +++++++++++++-----
 internal/metricdispatch/dataLoader_test.go    | 44 +++++-----
 internal/metricstoreclient/cc-metric-store.go | 56 ++++++-------
 internal/repository/stats.go                  |  2 +-
 internal/tagger/classifyJob.go                |  4 +-
 internal/tagger/classifyJob_test.go           | 12 +--
 .../taskmanager/updateFootprintService.go     |  4 +-
 pkg/archive/archive.go                        | 12 +--
 pkg/archive/fsBackend.go                      | 12 +--
 pkg/archive/fsBackend_test.go                 |  2 +-
 pkg/archive/json.go                           | 84 ++++++++++++-------
 pkg/archive/parquet/convert.go                |  2 +-
 pkg/archive/parquet/convert_test.go           | 24 +++---
 pkg/archive/parquet/writer_test.go            |  6 +-
 pkg/archive/s3Backend.go                      | 12 +--
 pkg/archive/sqliteBackend.go                  | 10 +--
 pkg/metricstore/query.go                      | 56 ++++++-------
 tools/archive-manager/import_test.go          |  4 +-
 25 files changed, 303 insertions(+), 196 deletions(-)

diff --git a/internal/api/api_test.go b/internal/api/api_test.go
index a8aef889..f736d73a 100644
--- a/internal/api/api_test.go
+++ b/internal/api/api_test.go
@@ -196,7 +196,7 @@ func cleanup() {
 func TestRestApi(t *testing.T) {
 	restapi := setup(t)
 	t.Cleanup(cleanup)
-	testData := schema.JobData{
+	testData := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
 		"load_one": map[schema.MetricScope]*schema.JobMetric{
 			schema.MetricScopeNode: {
 				Unit:     schema.Unit{Base: "load"},
@@ -210,7 +210,7 @@ func TestRestApi(t *testing.T) {
 				},
 			},
 		},
-	}
+	}}
 
 	metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
 		return testData, nil
@@ -497,7 +497,7 @@ func TestStopJobWithReusedJobId(t *testing.T) {
 	restapi := setup(t)
 	t.Cleanup(cleanup)
 
-	testData := schema.JobData{
+	testData := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
 		"load_one": map[schema.MetricScope]*schema.JobMetric{
 			schema.MetricScopeNode: {
 				Unit:     schema.Unit{Base: "load"},
@@ -511,7 +511,7 @@ func TestStopJobWithReusedJobId(t *testing.T) {
 				},
 			},
 		},
-	}
+	}}
 
 	metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
 		return testData, nil
diff --git a/internal/api/job.go b/internal/api/job.go
index 37c0e454..4816e0ee 100644
--- a/internal/api/job.go
+++ b/internal/api/job.go
@@ -412,7 +412,7 @@ func (api *RestAPI) getJobByID(rw http.ResponseWriter, r *http.Request) {
 	}
 
 	res := []*JobMetricWithName{}
-	for name, md := range data {
+	for name, md := range data.Metrics {
 		for scope, metric := range md {
 			res = append(res, &JobMetricWithName{
 				Name:   name,
diff --git a/internal/api/nats_test.go b/internal/api/nats_test.go
index b1d2a624..2ecd2e5b 100644
--- a/internal/api/nats_test.go
+++ b/internal/api/nats_test.go
@@ -531,7 +531,7 @@ func TestNatsHandleStopJob(t *testing.T) {
 		},
 	}
 
-	testData := schema.JobData{
+	testData := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
 		"load_one": map[schema.MetricScope]*schema.JobMetric{
 			schema.MetricScopeNode: {
 				Unit:     schema.Unit{Base: "load"},
@@ -545,7 +545,7 @@ func TestNatsHandleStopJob(t *testing.T) {
 				},
 			},
 		},
-	}
+	}}
 
 	metricstore.TestLoadDataCallback = func(job *schema.Job, metrics []string, scopes []schema.MetricScope, ctx context.Context, resolution int) (schema.JobData, error) {
 		return testData, nil
diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go
index 454a2358..22414c4d 100644
--- a/internal/archiver/archiver.go
+++ b/internal/archiver/archiver.go
@@ -65,11 +65,11 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
 		return nil, err
 	}
 
-	job.Statistics = make(map[string]schema.JobStatistics)
+	job.Statistics = schema.JobStatisticsSet{Metrics: make(map[string]schema.JobStatistics)}
 
-	for metric, data := range jobData {
+	for metric, data := range jobData.Metrics {
 		avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
-		nodeData, ok := data["node"]
+		nodeData, ok := data[schema.MetricScopeNode]
 		if !ok {
 			// This should never happen ?
 			continue
@@ -82,7 +82,7 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
 		}
 
 		// Round AVG Result to 2 Digits
-		job.Statistics[metric] = schema.JobStatistics{
+		job.Statistics.Metrics[metric] = schema.JobStatistics{
 			Unit: schema.Unit{
 				Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
 				Base:   archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
@@ -93,5 +93,43 @@ func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.Job, error) {
 		}
 	}
 
+	// Compute the same node-scope statistics for each array-valued metric group
+	// instance (e.g. per-filesystem) and attach it as a group in the statistics
+	// set so it round-trips into the job-meta "statistics" JSON.
+	for _, group := range jobData.Groups {
+		for _, inst := range group.Instances {
+			instStats := make(map[string]schema.JobStatistics)
+			for metric, data := range inst.Metrics {
+				avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32
+				nodeData, ok := data[schema.MetricScopeNode]
+				if !ok {
+					// This should never happen ?
+					continue
+				}
+
+				for _, series := range nodeData.Series {
+					avg += series.Statistics.Avg
+					min = math.Min(min, series.Statistics.Min)
+					max = math.Max(max, series.Statistics.Max)
+				}
+
+				// Prefer the unit from a MetricConfig if one exists, otherwise
+				// fall back to the unit already present on the JobMetric.
+				unit := nodeData.Unit
+				if mc := archive.GetMetricConfig(job.Cluster, metric); mc != nil {
+					unit = schema.Unit{Prefix: mc.Unit.Prefix, Base: mc.Unit.Base}
+				}
+
+				instStats[metric] = schema.JobStatistics{
+					Unit: unit,
+					Avg:  (math.Round((avg/float64(job.NumNodes))*100) / 100),
+					Min:  min,
+					Max:  max,
+				}
+			}
+			job.Statistics.AddGroupInstance(group.Key, inst.Name, inst.Type, instStats)
+		}
+	}
+
 	return job, archive.GetHandle().ImportJob(job, &jobData)
 }
diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go
index 3ef58ad4..679d6693 100644
--- a/internal/graph/schema.resolvers.go
+++ b/internal/graph/schema.resolvers.go
@@ -522,7 +522,7 @@ func (r *queryResolver) JobMetrics(ctx context.Context, id string, metrics []str
 	}
 
 	res := []*model.JobMetricWithName{}
-	for name, md := range data {
+	for name, md := range data.Metrics {
 		for scope, metric := range md {
 			res = append(res, &model.JobMetricWithName{
 				Name:   name,
@@ -575,7 +575,7 @@ func (r *queryResolver) ScopedJobStats(ctx context.Context, id string, metrics [
 	}
 
 	res := make([]*model.NamedStatsWithScope, 0)
-	for name, scoped := range data {
+	for name, scoped := range data.Metrics {
 		for scope, stats := range scoped {
 
 			mdlStats := make([]*model.ScopedStats, 0)
@@ -926,7 +926,7 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub
 			cclog.Warnf("error in nodeMetrics resolver: %s", err)
 		}
 
-		for metric, scopedMetrics := range data[hostname] {
+		for metric, scopedMetrics := range data[hostname].Metrics {
 			for scope, scopedMetric := range scopedMetrics {
 				host.Metrics = append(host.Metrics, &model.JobMetricWithName{
 					Name:   metric,
diff --git a/internal/graph/util.go b/internal/graph/util.go
index 5458d0ff..3a69f624 100644
--- a/internal/graph/util.go
+++ b/internal/graph/util.go
@@ -61,7 +61,7 @@ func (r *queryResolver) rooflineHeatmap(
 			return nil, err
 		}
 
-		flops_, membw_ := jobdata["flops_any"], jobdata["mem_bw"]
+		flops_, membw_ := jobdata.Metrics["flops_any"], jobdata.Metrics["mem_bw"]
 		if flops_ == nil && membw_ == nil {
 			cclog.Warnf("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", *job.ID)
 			continue
diff --git a/internal/importer/initDB.go b/internal/importer/initDB.go
index 87d92cd3..56076f84 100644
--- a/internal/importer/initDB.go
+++ b/internal/importer/initDB.go
@@ -290,7 +290,7 @@ func SanityChecks(job *schema.Job) error {
 //
 // TODO: Either implement the metric normalization or remove this dead code.
 func checkJobData(d *schema.JobData) error {
-	for _, scopes := range *d {
+	for _, scopes := range d.Metrics {
 		// var newUnit schema.Unit
 		// TODO Add node scope if missing
 		for _, metric := range scopes {
diff --git a/internal/metricdispatch/dataLoader.go b/internal/metricdispatch/dataLoader.go
index 2970f527..37dd2707 100644
--- a/internal/metricdispatch/dataLoader.go
+++ b/internal/metricdispatch/dataLoader.go
@@ -115,7 +115,7 @@ func LoadData(job *schema.Job,
 
 			jd, err = ms.LoadData(job, metrics, scopes, ctx, resolution)
 			if err != nil {
-				if len(jd) != 0 {
+				if len(jd.Metrics) != 0 {
 					cclog.Warnf("partial error loading metrics from store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
 						job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
 				} else {
@@ -138,33 +138,48 @@ func LoadData(job *schema.Job,
 
 			// Resample archived data using Largest Triangle Three Bucket algorithm to reduce data points
 			// to the requested resolution, improving transfer performance and client-side rendering.
-			for _, v := range jd {
-				for _, v_ := range v {
+			resampleScopes := func(scopeMap schema.ScopedMetrics) error {
+				for _, v_ := range scopeMap {
 					timestep := int64(0)
 					for i := 0; i < len(v_.Series); i += 1 {
 						v_.Series[i].Data, timestep, err = resampler.LargestTriangleThreeBucket(v_.Series[i].Data, int64(v_.Timestep), int64(resolution))
 						if err != nil {
-							return err, 0, 0
+							return err
 						}
 					}
 					v_.Timestep = int(timestep)
 				}
+				return nil
+			}
+			for _, v := range jd.Metrics {
+				if err := resampleScopes(v); err != nil {
+					return err, 0, 0
+				}
+			}
+			for _, group := range jd.Groups {
+				for _, inst := range group.Instances {
+					for _, v := range inst.Metrics {
+						if err := resampleScopes(v); err != nil {
+							return err, 0, 0
+						}
+					}
+				}
 			}
 
 			// Filter job data to only include requested metrics and scopes, avoiding unnecessary data transfer.
 			if metrics != nil || scopes != nil {
 				if metrics == nil {
-					metrics = make([]string, 0, len(jd))
-					for k := range jd {
+					metrics = make([]string, 0, len(jd.Metrics))
+					for k := range jd.Metrics {
 						metrics = append(metrics, k)
 					}
 				}
 
-				res := schema.JobData{}
+				res := schema.JobData{Metrics: make(map[string]schema.ScopedMetrics), Groups: jd.Groups}
 				for _, metric := range metrics {
-					if perscope, ok := jd[metric]; ok {
+					if perscope, ok := jd.Metrics[metric]; ok {
 						if len(perscope) > 1 {
-							subset := make(map[schema.MetricScope]*schema.JobMetric)
+							subset := make(schema.ScopedMetrics)
 							for _, scope := range scopes {
 								if jm, ok := perscope[scope]; ok {
 									subset[scope] = jm
@@ -176,7 +191,7 @@ func LoadData(job *schema.Job,
 							}
 						}
 
-						res[metric] = perscope
+						res.Metrics[metric] = perscope
 					}
 				}
 				jd = res
@@ -193,7 +208,7 @@ func LoadData(job *schema.Job,
 		// instead of overwhelming the UI with individual node lines. Note that newly calculated
 		// statistics use min/median/max, while archived statistics may use min/mean/max.
 		const maxSeriesSize int = 8
-		for _, scopes := range jd {
+		for _, scopes := range jd.Metrics {
 			for _, jm := range scopes {
 				if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
 					continue
@@ -223,7 +238,7 @@ func LoadData(job *schema.Job,
 
 	if err, ok := data.(error); ok {
 		cclog.Errorf("error in cached dataset for job %d: %s", job.JobID, err.Error())
-		return nil, err
+		return schema.JobData{}, err
 	}
 
 	return data.(schema.JobData), nil
@@ -290,14 +305,14 @@ func LoadScopedJobStats(
 	if err != nil {
 		cclog.Errorf("failed to access metricDataRepo for cluster %s-%s: %s",
 			job.Cluster, job.SubCluster, err.Error())
-		return nil, err
+		return schema.ScopedJobStats{}, err
 	}
 
 	scopedStats, err := ms.LoadScopedStats(job, metrics, scopes, ctx)
 	if err != nil {
 		cclog.Warnf("failed to load scoped statistics from metric store for job %d (user: %s, project: %s, cluster: %s-%s): %s",
 			job.JobID, job.User, job.Project, job.Cluster, job.SubCluster, err.Error())
-		return nil, err
+		return schema.ScopedJobStats{}, err
 	}
 
 	// Round Resulting Stat Values
@@ -444,7 +459,7 @@ func LoadNodeListData(
 	// Statistics are calculated as min/median/max.
 	const maxSeriesSize int = 8
 	for _, jd := range data {
-		for _, scopes := range jd {
+		for _, scopes := range jd.Metrics {
 			for _, jm := range scopes {
 				if jm.StatisticsSeries != nil || len(jm.Series) < maxSeriesSize {
 					continue
@@ -465,19 +480,47 @@ func LoadNodeListData(
 // archived data (e.g., during resampling). This ensures the cached archive data remains
 // immutable while allowing per-request transformations.
 func deepCopy(source schema.JobData) schema.JobData {
-	result := make(schema.JobData, len(source))
+	result := schema.JobData{Metrics: make(map[string]schema.ScopedMetrics, len(source.Metrics))}
 
-	for metricName, scopeMap := range source {
-		result[metricName] = make(map[schema.MetricScope]*schema.JobMetric, len(scopeMap))
+	for metricName, scopeMap := range source.Metrics {
+		result.Metrics[metricName] = copyScopedMetrics(scopeMap)
+	}
 
-		for scope, jobMetric := range scopeMap {
-			result[metricName][scope] = copyJobMetric(jobMetric)
+	// Deep-copy array-valued metric groups (e.g. filesystems) so their series are
+	// not shared with the cached archive data during resampling.
+	if len(source.Groups) > 0 {
+		result.Groups = make([]schema.MetricGroup, len(source.Groups))
+		for gi, group := range source.Groups {
+			dstGroup := schema.MetricGroup{
+				Key:       group.Key,
+				Instances: make([]schema.MetricGroupInstance, len(group.Instances)),
+			}
+			for ii, inst := range group.Instances {
+				dstInst := schema.MetricGroupInstance{
+					Name:    inst.Name,
+					Type:    inst.Type,
+					Metrics: make(map[string]schema.ScopedMetrics, len(inst.Metrics)),
+				}
+				for metricName, scopeMap := range inst.Metrics {
+					dstInst.Metrics[metricName] = copyScopedMetrics(scopeMap)
+				}
+				dstGroup.Instances[ii] = dstInst
+			}
+			result.Groups[gi] = dstGroup
 		}
 	}
 
 	return result
 }
 
+func copyScopedMetrics(scopeMap schema.ScopedMetrics) schema.ScopedMetrics {
+	dst := make(schema.ScopedMetrics, len(scopeMap))
+	for scope, jobMetric := range scopeMap {
+		dst[scope] = copyJobMetric(jobMetric)
+	}
+	return dst
+}
+
 func copyJobMetric(src *schema.JobMetric) *schema.JobMetric {
 	dst := &schema.JobMetric{
 		Timestep: src.Timestep,
diff --git a/internal/metricdispatch/dataLoader_test.go b/internal/metricdispatch/dataLoader_test.go
index 65a366f9..aadd8688 100644
--- a/internal/metricdispatch/dataLoader_test.go
+++ b/internal/metricdispatch/dataLoader_test.go
@@ -13,7 +13,7 @@ import (
 
 func TestDeepCopy(t *testing.T) {
 	nodeId := "0"
-	original := schema.JobData{
+	original := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
 		"cpu_load": {
 			schema.MetricScopeNode: &schema.JobMetric{
 				Timestep: 60,
@@ -42,42 +42,42 @@ func TestDeepCopy(t *testing.T) {
 				},
 			},
 		},
-	}
+	}}
 
 	copied := deepCopy(original)
 
-	original["cpu_load"][schema.MetricScopeNode].Series[0].Data[0] = 999.0
-	original["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0] = 888.0
-	original["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0] = 777.0
+	original.Metrics["cpu_load"][schema.MetricScopeNode].Series[0].Data[0] = 999.0
+	original.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0] = 888.0
+	original.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0] = 777.0
 
-	if copied["cpu_load"][schema.MetricScopeNode].Series[0].Data[0] != 1.0 {
+	if copied.Metrics["cpu_load"][schema.MetricScopeNode].Series[0].Data[0] != 1.0 {
 		t.Errorf("Series data was not deeply copied: got %v, want 1.0",
-			copied["cpu_load"][schema.MetricScopeNode].Series[0].Data[0])
+			copied.Metrics["cpu_load"][schema.MetricScopeNode].Series[0].Data[0])
 	}
 
-	if copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0] != 1.0 {
+	if copied.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0] != 1.0 {
 		t.Errorf("StatisticsSeries was not deeply copied: got %v, want 1.0",
-			copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0])
+			copied.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Min[0])
 	}
 
-	if copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0] != 1.5 {
+	if copied.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0] != 1.5 {
 		t.Errorf("Percentiles was not deeply copied: got %v, want 1.5",
-			copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0])
+			copied.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles[25][0])
 	}
 
-	if copied["cpu_load"][schema.MetricScopeNode].Timestep != 60 {
+	if copied.Metrics["cpu_load"][schema.MetricScopeNode].Timestep != 60 {
 		t.Errorf("Timestep not copied correctly: got %v, want 60",
-			copied["cpu_load"][schema.MetricScopeNode].Timestep)
+			copied.Metrics["cpu_load"][schema.MetricScopeNode].Timestep)
 	}
 
-	if copied["cpu_load"][schema.MetricScopeNode].Series[0].Hostname != "node001" {
+	if copied.Metrics["cpu_load"][schema.MetricScopeNode].Series[0].Hostname != "node001" {
 		t.Errorf("Hostname not copied correctly: got %v, want node001",
-			copied["cpu_load"][schema.MetricScopeNode].Series[0].Hostname)
+			copied.Metrics["cpu_load"][schema.MetricScopeNode].Series[0].Hostname)
 	}
 }
 
 func TestDeepCopyNilStatisticsSeries(t *testing.T) {
-	original := schema.JobData{
+	original := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
 		"mem_used": {
 			schema.MetricScopeNode: &schema.JobMetric{
 				Timestep: 60,
@@ -90,18 +90,18 @@ func TestDeepCopyNilStatisticsSeries(t *testing.T) {
 				StatisticsSeries: nil,
 			},
 		},
-	}
+	}}
 
 	copied := deepCopy(original)
 
-	if copied["mem_used"][schema.MetricScopeNode].StatisticsSeries != nil {
+	if copied.Metrics["mem_used"][schema.MetricScopeNode].StatisticsSeries != nil {
 		t.Errorf("StatisticsSeries should be nil, got %v",
-			copied["mem_used"][schema.MetricScopeNode].StatisticsSeries)
+			copied.Metrics["mem_used"][schema.MetricScopeNode].StatisticsSeries)
 	}
 }
 
 func TestDeepCopyEmptyPercentiles(t *testing.T) {
-	original := schema.JobData{
+	original := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
 		"cpu_load": {
 			schema.MetricScopeNode: &schema.JobMetric{
 				Timestep: 60,
@@ -115,11 +115,11 @@ func TestDeepCopyEmptyPercentiles(t *testing.T) {
 				},
 			},
 		},
-	}
+	}}
 
 	copied := deepCopy(original)
 
-	if copied["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles != nil {
+	if copied.Metrics["cpu_load"][schema.MetricScopeNode].StatisticsSeries.Percentiles != nil {
 		t.Errorf("Percentiles should be nil when source is nil/empty")
 	}
 }
diff --git a/internal/metricstoreclient/cc-metric-store.go b/internal/metricstoreclient/cc-metric-store.go
index 55dc7fb5..d3efe7c5 100644
--- a/internal/metricstoreclient/cc-metric-store.go
+++ b/internal/metricstoreclient/cc-metric-store.go
@@ -235,7 +235,7 @@ func (ccms *CCMetricStore) LoadData(
 	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, resolution)
 	if err != nil {
 		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
-		return nil, err
+		return schema.JobData{}, err
 	}
 
 	// Verify assignment is correct - log any inconsistencies for debugging
@@ -256,11 +256,11 @@ func (ccms *CCMetricStore) LoadData(
 	resBody, err := ccms.doRequest(ctx, &req)
 	if err != nil {
 		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
-		return nil, err
+		return schema.JobData{}, err
 	}
 
 	var errors []string
-	jobData := make(schema.JobData)
+	jobData := schema.JobData{Metrics: make(map[string]schema.ScopedMetrics)}
 
 	// Add safety check for potential index out of range errors
 	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
@@ -285,8 +285,8 @@ func (ccms *CCMetricStore) LoadData(
 			continue
 		}
 
-		if _, ok := jobData[metric]; !ok {
-			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
+		if _, ok := jobData.Metrics[metric]; !ok {
+			jobData.Metrics[metric] = make(schema.ScopedMetrics)
 		}
 
 		res := mc.Timestep
@@ -294,14 +294,14 @@ func (ccms *CCMetricStore) LoadData(
 			res = row[0].Resolution
 		}
 
-		jobMetric, ok := jobData[metric][scope]
+		jobMetric, ok := jobData.Metrics[metric][scope]
 		if !ok {
 			jobMetric = &schema.JobMetric{
 				Unit:     mc.Unit,
 				Timestep: res,
 				Series:   make([]schema.Series, 0),
 			}
-			jobData[metric][scope] = jobMetric
+			jobData.Metrics[metric][scope] = jobMetric
 		}
 
 		for ndx, res := range row {
@@ -329,9 +329,9 @@ func (ccms *CCMetricStore) LoadData(
 
 		// So that one can later check len(jobData):
 		if len(jobMetric.Series) == 0 {
-			delete(jobData[metric], scope)
-			if len(jobData[metric]) == 0 {
-				delete(jobData, metric)
+			delete(jobData.Metrics[metric], scope)
+			if len(jobData.Metrics[metric]) == 0 {
+				delete(jobData.Metrics, metric)
 			}
 		}
 	}
@@ -426,7 +426,7 @@ func (ccms *CCMetricStore) LoadScopedStats(
 	queries, assignedScope, err := ccms.buildQueries(job, metrics, scopes, 0)
 	if err != nil {
 		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
-		return nil, err
+		return schema.ScopedJobStats{}, err
 	}
 
 	req := APIQueryRequest{
@@ -441,23 +441,23 @@ func (ccms *CCMetricStore) LoadScopedStats(
 	resBody, err := ccms.doRequest(ctx, &req)
 	if err != nil {
 		cclog.Errorf("Error while performing request for job %d: %s", job.JobID, err.Error())
-		return nil, err
+		return schema.ScopedJobStats{}, err
 	}
 
 	var errors []string
-	scopedJobStats := make(schema.ScopedJobStats)
+	scopedJobStats := schema.ScopedJobStats{Metrics: make(map[string]schema.ScopedMetricStats)}
 
 	for i, row := range resBody.Results {
 		query := req.Queries[i]
 		metric := query.Metric
 		scope := assignedScope[i]
 
-		if _, ok := scopedJobStats[metric]; !ok {
-			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
+		if _, ok := scopedJobStats.Metrics[metric]; !ok {
+			scopedJobStats.Metrics[metric] = make(schema.ScopedMetricStats)
 		}
 
-		if _, ok := scopedJobStats[metric][scope]; !ok {
-			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
+		if _, ok := scopedJobStats.Metrics[metric][scope]; !ok {
+			scopedJobStats.Metrics[metric][scope] = make([]*schema.ScopedStats, 0)
 		}
 
 		for ndx, res := range row {
@@ -471,7 +471,7 @@ func (ccms *CCMetricStore) LoadScopedStats(
 
 			ms.SanitizeStats(&res.Avg, &res.Min, &res.Max)
 
-			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
+			scopedJobStats.Metrics[metric][scope] = append(scopedJobStats.Metrics[metric][scope], &schema.ScopedStats{
 				Hostname: query.Hostname,
 				ID:       id,
 				Data: &schema.MetricStatistics{
@@ -482,11 +482,11 @@ func (ccms *CCMetricStore) LoadScopedStats(
 			})
 		}
 
-		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
-		if len(scopedJobStats[metric][scope]) == 0 {
-			delete(scopedJobStats[metric], scope)
-			if len(scopedJobStats[metric]) == 0 {
-				delete(scopedJobStats, metric)
+		// So that one can later check len(scopedJobStats.Metrics[metric][scope]): Remove from map if empty
+		if len(scopedJobStats.Metrics[metric][scope]) == 0 {
+			delete(scopedJobStats.Metrics[metric], scope)
+			if len(scopedJobStats.Metrics[metric]) == 0 {
+				delete(scopedJobStats.Metrics, metric)
 			}
 		}
 	}
@@ -690,14 +690,14 @@ func (ccms *CCMetricStore) LoadNodeListData(
 		// Init Nested Map Data Structures If Not Found
 		hostData, ok := data[query.Hostname]
 		if !ok {
-			hostData = make(schema.JobData)
+			hostData = schema.JobData{Metrics: make(map[string]schema.ScopedMetrics)}
 			data[query.Hostname] = hostData
 		}
 
-		metricData, ok := hostData[metric]
+		metricData, ok := hostData.Metrics[metric]
 		if !ok {
-			metricData = make(map[schema.MetricScope]*schema.JobMetric)
-			data[query.Hostname][metric] = metricData
+			metricData = make(schema.ScopedMetrics)
+			data[query.Hostname].Metrics[metric] = metricData
 		}
 
 		scopeData, ok := metricData[scope]
@@ -707,7 +707,7 @@ func (ccms *CCMetricStore) LoadNodeListData(
 				Timestep: res,
 				Series:   make([]schema.Series, 0),
 			}
-			data[query.Hostname][metric][scope] = scopeData
+			data[query.Hostname].Metrics[metric][scope] = scopeData
 		}
 
 		for ndx, res := range row {
diff --git a/internal/repository/stats.go b/internal/repository/stats.go
index fb0bc075..7fea939b 100644
--- a/internal/repository/stats.go
+++ b/internal/repository/stats.go
@@ -403,7 +403,7 @@ func (r *JobRepository) JobsStats(
 //
 // Returns the requested statistic value or 0.0 if not found.
 func LoadJobStat(job *schema.Job, metric string, statType string) float64 {
-	if stats, ok := job.Statistics[metric]; ok {
+	if stats, ok := job.Statistics.Metrics[metric]; ok {
 		switch statType {
 		case "avg":
 			return stats.Avg
diff --git a/internal/tagger/classifyJob.go b/internal/tagger/classifyJob.go
index 81698337..9e72e58b 100644
--- a/internal/tagger/classifyJob.go
+++ b/internal/tagger/classifyJob.go
@@ -107,7 +107,7 @@ type JobClassTagger struct {
 	// repo provides access to job database operations
 	repo JobRepository
 	// getStatistics retrieves job statistics for analysis
-	getStatistics func(job *schema.Job) (map[string]schema.JobStatistics, error)
+	getStatistics func(job *schema.Job) (schema.JobStatisticsSet, error)
 	// getMetricConfig retrieves metric configuration (limits) for a cluster
 	getMetricConfig func(cluster, subCluster string) map[string]*schema.Metric
 }
@@ -361,7 +361,7 @@ func (t *JobClassTagger) Match(job *schema.Job) {
 		// add metrics to env
 		skipRule := false
 		for _, m := range ri.metrics {
-			stats, ok := jobStats[m]
+			stats, ok := jobStats.Metrics[m]
 			if !ok {
 				cclog.Debugf("job classification: missing metric '%s' for rule %s on job %d", m, tag, job.JobID)
 				skipRule = true
diff --git a/internal/tagger/classifyJob_test.go b/internal/tagger/classifyJob_test.go
index f82cf807..8f051ed4 100644
--- a/internal/tagger/classifyJob_test.go
+++ b/internal/tagger/classifyJob_test.go
@@ -64,10 +64,10 @@ func TestClassifyJobMatch(t *testing.T) {
 		parameters: make(map[string]any),
 		tagType:    "jobClass",
 		repo:       mockRepo,
-		getStatistics: func(job *schema.Job) (map[string]schema.JobStatistics, error) {
-			return map[string]schema.JobStatistics{
+		getStatistics: func(job *schema.Job) (schema.JobStatisticsSet, error) {
+			return schema.JobStatisticsSet{Metrics: map[string]schema.JobStatistics{
 				"flops_any": {Min: 0, Max: 200, Avg: 150},
-			}, nil
+			}}, nil
 		},
 		getMetricConfig: func(cluster, subCluster string) map[string]*schema.Metric {
 			return map[string]*schema.Metric{
@@ -120,10 +120,10 @@ func TestMatch_NoMatch(t *testing.T) {
 		parameters: make(map[string]any),
 		tagType:    "jobClass",
 		repo:       mockRepo,
-		getStatistics: func(job *schema.Job) (map[string]schema.JobStatistics, error) {
-			return map[string]schema.JobStatistics{
+		getStatistics: func(job *schema.Job) (schema.JobStatisticsSet, error) {
+			return schema.JobStatisticsSet{Metrics: map[string]schema.JobStatistics{
 				"flops_any": {Min: 0, Max: 50, Avg: 20}, // Avg 20 < 100
-			}, nil
+			}}, nil
 		},
 		getMetricConfig: func(cluster, subCluster string) map[string]*schema.Metric {
 			return map[string]*schema.Metric{
diff --git a/internal/taskmanager/updateFootprintService.go b/internal/taskmanager/updateFootprintService.go
index 34a18bdd..d2fa194e 100644
--- a/internal/taskmanager/updateFootprintService.go
+++ b/internal/taskmanager/updateFootprintService.go
@@ -80,7 +80,7 @@ func RegisterFootprintWorker() {
 							continue
 						}
 
-						job.Statistics = make(map[string]schema.JobStatistics)
+						job.Statistics = schema.JobStatisticsSet{Metrics: make(map[string]schema.JobStatistics)}
 
 						for _, metric := range allMetrics {
 							avg, min, max := 0.0, 0.0, 0.0
@@ -98,7 +98,7 @@ func RegisterFootprintWorker() {
 							}
 
 							// Add values rounded to 2 digits: repo.LoadStats may return unrounded
-							job.Statistics[metric] = schema.JobStatistics{
+							job.Statistics.Metrics[metric] = schema.JobStatistics{
 								Unit: schema.Unit{
 									Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix,
 									Base:   archive.GetMetricConfig(job.Cluster, metric).Unit.Base,
diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go
index f993f025..07e8a8bf 100644
--- a/pkg/archive/archive.go
+++ b/pkg/archive/archive.go
@@ -299,7 +299,7 @@ func LoadAveragesFromArchive(
 	}
 
 	for i, m := range metrics {
-		if stat, ok := metaFile.Statistics[m]; ok {
+		if stat, ok := metaFile.Statistics.Metrics[m]; ok {
 			data[i] = append(data[i], schema.Float(stat.Avg))
 		} else {
 			data[i] = append(data[i], schema.NaN)
@@ -323,7 +323,7 @@ func LoadStatsFromArchive(
 	}
 
 	for _, m := range metrics {
-		stat, ok := metaFile.Statistics[m]
+		stat, ok := metaFile.Statistics.Metrics[m]
 		if !ok {
 			data[m] = schema.MetricStatistics{Min: 0.0, Avg: 0.0, Max: 0.0}
 			continue
@@ -349,19 +349,19 @@ func LoadScopedStatsFromArchive(
 	data, err := ar.LoadJobStats(job)
 	if err != nil {
 		cclog.Errorf("Error while loading job stats from archiveBackend: %s", err.Error())
-		return nil, err
+		return schema.ScopedJobStats{}, err
 	}
 
 	return data, nil
 }
 
 // GetStatistics returns all metric statistics for a job.
-// Returns a map of metric names to their job-level statistics.
-func GetStatistics(job *schema.Job) (map[string]schema.JobStatistics, error) {
+// Returns the job-level statistics set.
+func GetStatistics(job *schema.Job) (schema.JobStatisticsSet, error) {
 	metaFile, err := ar.LoadJobMeta(job)
 	if err != nil {
 		cclog.Errorf("Error while loading job metadata from archiveBackend: %s", err.Error())
-		return nil, err
+		return schema.JobStatisticsSet{}, err
 	}
 
 	return metaFile.Statistics, nil
diff --git a/pkg/archive/fsBackend.go b/pkg/archive/fsBackend.go
index dfc870b4..228f720d 100644
--- a/pkg/archive/fsBackend.go
+++ b/pkg/archive/fsBackend.go
@@ -93,7 +93,7 @@ func loadJobData(filename string, isCompressed bool) (schema.JobData, error) {
 	f, err := os.Open(filename)
 	if err != nil {
 		cclog.Errorf("fsBackend LoadJobData()- %v", err)
-		return nil, err
+		return schema.JobData{}, err
 	}
 	defer f.Close()
 
@@ -101,7 +101,7 @@ func loadJobData(filename string, isCompressed bool) (schema.JobData, error) {
 		r, err := gzip.NewReader(f)
 		if err != nil {
 			cclog.Errorf(" %v", err)
-			return nil, err
+			return schema.JobData{}, err
 		}
 		defer r.Close()
 
@@ -126,7 +126,7 @@ func loadJobStats(filename string, isCompressed bool) (schema.ScopedJobStats, er
 	f, err := os.Open(filename)
 	if err != nil {
 		cclog.Errorf("fsBackend LoadJobStats()- %v", err)
-		return nil, err
+		return schema.ScopedJobStats{}, err
 	}
 	defer f.Close()
 
@@ -134,13 +134,13 @@ func loadJobStats(filename string, isCompressed bool) (schema.ScopedJobStats, er
 		r, err := gzip.NewReader(f)
 		if err != nil {
 			cclog.Errorf(" %v", err)
-			return nil, err
+			return schema.ScopedJobStats{}, err
 		}
 		defer r.Close()
 
 		if config.Keys.Validate {
 			if err := schema.Validate(schema.Data, r); err != nil {
-				return nil, fmt.Errorf("validate job data: %v", err)
+				return schema.ScopedJobStats{}, fmt.Errorf("validate job data: %v", err)
 			}
 		}
 
@@ -148,7 +148,7 @@ func loadJobStats(filename string, isCompressed bool) (schema.ScopedJobStats, er
 	} else {
 		if config.Keys.Validate {
 			if err := schema.Validate(schema.Data, bufio.NewReader(f)); err != nil {
-				return nil, fmt.Errorf("validate job data: %v", err)
+				return schema.ScopedJobStats{}, fmt.Errorf("validate job data: %v", err)
 			}
 		}
 		return DecodeJobStats(bufio.NewReader(f), filename)
diff --git a/pkg/archive/fsBackend_test.go b/pkg/archive/fsBackend_test.go
index 05491f61..6d63a6fc 100644
--- a/pkg/archive/fsBackend_test.go
+++ b/pkg/archive/fsBackend_test.go
@@ -129,7 +129,7 @@ func TestLoadJobData(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, scopes := range data {
+	for _, scopes := range data.Metrics {
 		// fmt.Printf("Metric name: %s\n", name)
 
 		if _, exists := scopes[schema.MetricScopeNode]; !exists {
diff --git a/pkg/archive/json.go b/pkg/archive/json.go
index dd37075d..d4c95434 100644
--- a/pkg/archive/json.go
+++ b/pkg/archive/json.go
@@ -27,47 +27,73 @@ func DecodeJobData(r io.Reader, k string) (schema.JobData, error) {
 
 	if err, ok := data.(error); ok {
 		cclog.Warn("Error in decoded job data set")
-		return nil, err
+		return schema.JobData{}, err
 	}
 
 	return data.(schema.JobData), nil
 }
 
-func DecodeJobStats(r io.Reader, k string) (schema.ScopedJobStats, error) {
-	jobData, err := DecodeJobData(r, k)
-	// Convert schema.JobData to schema.ScopedJobStats
-	if jobData != nil {
-		scopedJobStats := make(schema.ScopedJobStats)
-		for metric, metricData := range jobData {
-			if _, ok := scopedJobStats[metric]; !ok {
-				scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
+// scopedStatsFromMetrics converts a flat metric->scope->JobMetric map into the
+// corresponding metric->scope->[]*ScopedStats map. It preserves the "remove
+// empty" cleanup behaviour: a scope with no series is dropped, and a metric
+// with no remaining scopes is dropped.
+func scopedStatsFromMetrics(metrics map[string]schema.ScopedMetrics) map[string]schema.ScopedMetricStats {
+	result := make(map[string]schema.ScopedMetricStats)
+	for metric, metricData := range metrics {
+		if _, ok := result[metric]; !ok {
+			result[metric] = make(schema.ScopedMetricStats)
+		}
+
+		for scope, jobMetric := range metricData {
+			if _, ok := result[metric][scope]; !ok {
+				result[metric][scope] = make([]*schema.ScopedStats, 0)
 			}
 
-			for scope, jobMetric := range metricData {
-				if _, ok := scopedJobStats[metric][scope]; !ok {
-					scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
-				}
+			for _, series := range jobMetric.Series {
+				result[metric][scope] = append(result[metric][scope], &schema.ScopedStats{
+					Hostname: series.Hostname,
+					ID:       series.ID,
+					Data:     &series.Statistics,
+				})
+			}
 
-				for _, series := range jobMetric.Series {
-					scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
-						Hostname: series.Hostname,
-						ID:       series.ID,
-						Data:     &series.Statistics,
-					})
-				}
-
-				// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
-				if len(scopedJobStats[metric][scope]) == 0 {
-					delete(scopedJobStats[metric], scope)
-					if len(scopedJobStats[metric]) == 0 {
-						delete(scopedJobStats, metric)
-					}
+			// So that one can later check len(result[metric][scope]): Remove from map if empty
+			if len(result[metric][scope]) == 0 {
+				delete(result[metric], scope)
+				if len(result[metric]) == 0 {
+					delete(result, metric)
 				}
 			}
 		}
-		return scopedJobStats, nil
 	}
-	return nil, err
+	return result
+}
+
+func DecodeJobStats(r io.Reader, k string) (schema.ScopedJobStats, error) {
+	jobData, err := DecodeJobData(r, k)
+	if err != nil {
+		return schema.ScopedJobStats{}, err
+	}
+
+	// Convert schema.JobData to schema.ScopedJobStats
+	scopedJobStats := schema.ScopedJobStats{
+		Metrics: scopedStatsFromMetrics(jobData.Metrics),
+	}
+
+	// Project array-valued metric groups (e.g. filesystems) analogously.
+	for _, group := range jobData.Groups {
+		scopedGroup := schema.ScopedStatsGroup{Key: group.Key}
+		for _, inst := range group.Instances {
+			scopedGroup.Instances = append(scopedGroup.Instances, schema.ScopedStatsGroupInstance{
+				Name:    inst.Name,
+				Type:    inst.Type,
+				Metrics: scopedStatsFromMetrics(inst.Metrics),
+			})
+		}
+		scopedJobStats.Groups = append(scopedJobStats.Groups, scopedGroup)
+	}
+
+	return scopedJobStats, nil
 }
 
 func DecodeJobMeta(r io.Reader) (*schema.Job, error) {
diff --git a/pkg/archive/parquet/convert.go b/pkg/archive/parquet/convert.go
index 43e611e4..4e1cb4a0 100644
--- a/pkg/archive/parquet/convert.go
+++ b/pkg/archive/parquet/convert.go
@@ -23,7 +23,7 @@ func JobToParquetRow(meta *schema.Job, data *schema.JobData) (*ParquetJobRow, er
 	}
 
 	var statisticsJSON []byte
-	if meta.Statistics != nil {
+	if meta.Statistics.Metrics != nil {
 		statisticsJSON, err = json.Marshal(meta.Statistics)
 		if err != nil {
 			return nil, fmt.Errorf("marshal statistics: %w", err)
diff --git a/pkg/archive/parquet/convert_test.go b/pkg/archive/parquet/convert_test.go
index 3b2848ba..ee40b52f 100644
--- a/pkg/archive/parquet/convert_test.go
+++ b/pkg/archive/parquet/convert_test.go
@@ -32,9 +32,9 @@ func TestParquetRowToJob(t *testing.T) {
 			{Hostname: "node001", HWThreads: []int{0, 1, 2, 3}},
 			{Hostname: "node002", HWThreads: []int{4, 5, 6, 7}},
 		},
-		Statistics: map[string]schema.JobStatistics{
+		Statistics: schema.JobStatisticsSet{Metrics: map[string]schema.JobStatistics{
 			"cpu_load": {Avg: 50.0, Min: 10.0, Max: 90.0},
-		},
+		}},
 		Tags: []*schema.Tag{
 			{Type: "test", Name: "tag1"},
 		},
@@ -49,7 +49,7 @@ func TestParquetRowToJob(t *testing.T) {
 		},
 	}
 
-	data := &schema.JobData{
+	data := &schema.JobData{Metrics: map[string]schema.ScopedMetrics{
 		"cpu_load": {
 			schema.MetricScopeNode: &schema.JobMetric{
 				Unit:     schema.Unit{Base: ""},
@@ -62,7 +62,7 @@ func TestParquetRowToJob(t *testing.T) {
 				},
 			},
 		},
-	}
+	}}
 
 	// Convert to parquet row
 	row, err := JobToParquetRow(meta, data)
@@ -134,10 +134,10 @@ func TestParquetRowToJob(t *testing.T) {
 		t.Errorf("Resources[0].HWThreads len = %d, want 4", len(gotMeta.Resources[0].HWThreads))
 	}
 
-	if len(gotMeta.Statistics) != 1 {
-		t.Fatalf("Statistics len = %d, want 1", len(gotMeta.Statistics))
+	if len(gotMeta.Statistics.Metrics) != 1 {
+		t.Fatalf("Statistics len = %d, want 1", len(gotMeta.Statistics.Metrics))
 	}
-	if stat, ok := gotMeta.Statistics["cpu_load"]; !ok {
+	if stat, ok := gotMeta.Statistics.Metrics["cpu_load"]; !ok {
 		t.Error("Statistics missing cpu_load")
 	} else if stat.Avg != 50.0 {
 		t.Errorf("Statistics[cpu_load].Avg = %f, want 50.0", stat.Avg)
@@ -163,7 +163,7 @@ func TestParquetRowToJob(t *testing.T) {
 	if gotData == nil {
 		t.Fatal("JobData is nil")
 	}
-	cpuLoad, ok := (*gotData)["cpu_load"]
+	cpuLoad, ok := (*gotData).Metrics["cpu_load"]
 	if !ok {
 		t.Fatal("JobData missing cpu_load")
 	}
@@ -201,7 +201,7 @@ func TestParquetRowToJobNilOptionalFields(t *testing.T) {
 		},
 	}
 
-	data := &schema.JobData{
+	data := &schema.JobData{Metrics: map[string]schema.ScopedMetrics{
 		"cpu_load": {
 			schema.MetricScopeNode: &schema.JobMetric{
 				Timestep: 60,
@@ -210,7 +210,7 @@ func TestParquetRowToJobNilOptionalFields(t *testing.T) {
 				},
 			},
 		},
-	}
+	}}
 
 	row, err := JobToParquetRow(meta, data)
 	if err != nil {
@@ -228,7 +228,7 @@ func TestParquetRowToJobNilOptionalFields(t *testing.T) {
 	if gotMeta.Tags != nil {
 		t.Errorf("Tags should be nil, got %v", gotMeta.Tags)
 	}
-	if gotMeta.Statistics != nil {
+	if gotMeta.Statistics.Metrics != nil {
 		t.Errorf("Statistics should be nil, got %v", gotMeta.Statistics)
 	}
 	if gotMeta.MetaData != nil {
@@ -299,7 +299,7 @@ func TestRoundTripThroughParquetFile(t *testing.T) {
 	if gotData == nil {
 		t.Fatal("JobData is nil")
 	}
-	if _, ok := (*gotData)["cpu_load"]; !ok {
+	if _, ok := (*gotData).Metrics["cpu_load"]; !ok {
 		t.Error("JobData missing cpu_load")
 	}
 }
diff --git a/pkg/archive/parquet/writer_test.go b/pkg/archive/parquet/writer_test.go
index 9515edc3..67805b78 100644
--- a/pkg/archive/parquet/writer_test.go
+++ b/pkg/archive/parquet/writer_test.go
@@ -57,7 +57,7 @@ func makeTestJob(jobID int64) (*schema.Job, *schema.JobData) {
 		},
 	}
 
-	data := schema.JobData{
+	data := schema.JobData{Metrics: map[string]schema.ScopedMetrics{
 		"cpu_load": {
 			schema.MetricScopeNode: &schema.JobMetric{
 				Unit:     schema.Unit{Base: ""},
@@ -70,7 +70,7 @@ func makeTestJob(jobID int64) (*schema.Job, *schema.JobData) {
 				},
 			},
 		},
-	}
+	}}
 
 	return meta, &data
 }
@@ -132,7 +132,7 @@ func TestJobToParquetRowConversion(t *testing.T) {
 	if err := json.Unmarshal(decompressed, &jobData); err != nil {
 		t.Fatalf("unmarshal metric data: %v", err)
 	}
-	if _, ok := jobData["cpu_load"]; !ok {
+	if _, ok := jobData.Metrics["cpu_load"]; !ok {
 		t.Error("metric data missing cpu_load key")
 	}
 }
diff --git a/pkg/archive/s3Backend.go b/pkg/archive/s3Backend.go
index 7b82d309..25db2309 100644
--- a/pkg/archive/s3Backend.go
+++ b/pkg/archive/s3Backend.go
@@ -330,7 +330,7 @@ func (s3a *S3Archive) LoadJobData(job *schema.Job) (schema.JobData, error) {
 		})
 		if err != nil {
 			cclog.Errorf("S3Archive LoadJobData() > GetObject error: %v", err)
-			return nil, err
+			return schema.JobData{}, err
 		}
 		defer result.Body.Close()
 
@@ -349,7 +349,7 @@ func (s3a *S3Archive) LoadJobData(job *schema.Job) (schema.JobData, error) {
 	r, err := gzip.NewReader(result.Body)
 	if err != nil {
 		cclog.Errorf("S3Archive LoadJobData() > gzip error: %v", err)
-		return nil, err
+		return schema.JobData{}, err
 	}
 	defer r.Close()
 
@@ -381,14 +381,14 @@ func (s3a *S3Archive) LoadJobStats(job *schema.Job) (schema.ScopedJobStats, erro
 		})
 		if err != nil {
 			cclog.Errorf("S3Archive LoadJobStats() > GetObject error: %v", err)
-			return nil, err
+			return schema.ScopedJobStats{}, err
 		}
 		defer result.Body.Close()
 
 		if config.Keys.Validate {
 			b, _ := io.ReadAll(result.Body)
 			if err := schema.Validate(schema.Data, bytes.NewReader(b)); err != nil {
-				return nil, fmt.Errorf("validate job data: %v", err)
+				return schema.ScopedJobStats{}, fmt.Errorf("validate job data: %v", err)
 			}
 			return DecodeJobStats(bytes.NewReader(b), key)
 		}
@@ -400,14 +400,14 @@ func (s3a *S3Archive) LoadJobStats(job *schema.Job) (schema.ScopedJobStats, erro
 	r, err := gzip.NewReader(result.Body)
 	if err != nil {
 		cclog.Errorf("S3Archive LoadJobStats() > gzip error: %v", err)
-		return nil, err
+		return schema.ScopedJobStats{}, err
 	}
 	defer r.Close()
 
 	if config.Keys.Validate {
 		b, _ := io.ReadAll(r)
 		if err := schema.Validate(schema.Data, bytes.NewReader(b)); err != nil {
-			return nil, fmt.Errorf("validate job data: %v", err)
+			return schema.ScopedJobStats{}, fmt.Errorf("validate job data: %v", err)
 		}
 		return DecodeJobStats(bytes.NewReader(b), keyGz)
 	}
diff --git a/pkg/archive/sqliteBackend.go b/pkg/archive/sqliteBackend.go
index ff7bf333..3cb33ef3 100644
--- a/pkg/archive/sqliteBackend.go
+++ b/pkg/archive/sqliteBackend.go
@@ -252,7 +252,7 @@ func (sa *SqliteArchive) LoadJobData(job *schema.Job) (schema.JobData, error) {
 		job.JobID, job.Cluster, job.StartTime).Scan(&dataBlob, &compressed)
 	if err != nil {
 		cclog.Errorf("SqliteArchive LoadJobData() > query error: %v", err)
-		return nil, err
+		return schema.JobData{}, err
 	}
 	key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime)
 
@@ -261,7 +261,7 @@ func (sa *SqliteArchive) LoadJobData(job *schema.Job) (schema.JobData, error) {
 		gzipReader, err := gzip.NewReader(reader)
 		if err != nil {
 			cclog.Errorf("SqliteArchive LoadJobData() > gzip error: %v", err)
-			return nil, err
+			return schema.JobData{}, err
 		}
 		defer gzipReader.Close()
 		reader = gzipReader
@@ -285,7 +285,7 @@ func (sa *SqliteArchive) LoadJobStats(job *schema.Job) (schema.ScopedJobStats, e
 		job.JobID, job.Cluster, job.StartTime).Scan(&dataBlob, &compressed)
 	if err != nil {
 		cclog.Errorf("SqliteArchive LoadJobStats() > query error: %v", err)
-		return nil, err
+		return schema.ScopedJobStats{}, err
 	}
 	key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime)
 
@@ -294,7 +294,7 @@ func (sa *SqliteArchive) LoadJobStats(job *schema.Job) (schema.ScopedJobStats, e
 		gzipReader, err := gzip.NewReader(reader)
 		if err != nil {
 			cclog.Errorf("SqliteArchive LoadJobStats() > gzip error: %v", err)
-			return nil, err
+			return schema.ScopedJobStats{}, err
 		}
 		defer gzipReader.Close()
 		reader = gzipReader
@@ -303,7 +303,7 @@ func (sa *SqliteArchive) LoadJobStats(job *schema.Job) (schema.ScopedJobStats, e
 	if config.Keys.Validate {
 		data, _ := io.ReadAll(reader)
 		if err := schema.Validate(schema.Data, bytes.NewReader(data)); err != nil {
-			return nil, fmt.Errorf("validate job data: %v", err)
+			return schema.ScopedJobStats{}, fmt.Errorf("validate job data: %v", err)
 		}
 		return DecodeJobStats(bytes.NewReader(data), key)
 	}
diff --git a/pkg/metricstore/query.go b/pkg/metricstore/query.go
index 8a349b5a..762c7d64 100644
--- a/pkg/metricstore/query.go
+++ b/pkg/metricstore/query.go
@@ -89,7 +89,7 @@ func (ccms *InternalMetricStore) LoadData(
 	queries, assignedScope, err := buildQueries(job, metrics, scopes, int64(resolution))
 	if err != nil {
 		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
-		return nil, err
+		return schema.JobData{}, err
 	}
 
 	// Verify assignment is correct - log any inconsistencies for debugging
@@ -110,11 +110,11 @@ func (ccms *InternalMetricStore) LoadData(
 	resBody, err := FetchData(req)
 	if err != nil {
 		cclog.Errorf("Error while fetching data : %s", err.Error())
-		return nil, err
+		return schema.JobData{}, err
 	}
 
 	var errors []string
-	jobData := make(schema.JobData)
+	jobData := schema.JobData{Metrics: make(map[string]schema.ScopedMetrics)}
 
 	// Add safety check for potential index out of range errors
 	if len(resBody.Results) != len(req.Queries) || len(assignedScope) != len(req.Queries) {
@@ -139,8 +139,8 @@ func (ccms *InternalMetricStore) LoadData(
 			continue
 		}
 
-		if _, ok := jobData[metric]; !ok {
-			jobData[metric] = make(map[schema.MetricScope]*schema.JobMetric)
+		if _, ok := jobData.Metrics[metric]; !ok {
+			jobData.Metrics[metric] = make(schema.ScopedMetrics)
 		}
 
 		res := mc.Timestep
@@ -148,14 +148,14 @@ func (ccms *InternalMetricStore) LoadData(
 			res = int(row[0].Resolution)
 		}
 
-		jobMetric, ok := jobData[metric][scope]
+		jobMetric, ok := jobData.Metrics[metric][scope]
 		if !ok {
 			jobMetric = &schema.JobMetric{
 				Unit:     mc.Unit,
 				Timestep: res,
 				Series:   make([]schema.Series, 0),
 			}
-			jobData[metric][scope] = jobMetric
+			jobData.Metrics[metric][scope] = jobMetric
 		}
 
 		for ndx, res := range row {
@@ -183,9 +183,9 @@ func (ccms *InternalMetricStore) LoadData(
 
 		// So that one can later check len(jobData):
 		if len(jobMetric.Series) == 0 {
-			delete(jobData[metric], scope)
-			if len(jobData[metric]) == 0 {
-				delete(jobData, metric)
+			delete(jobData.Metrics[metric], scope)
+			if len(jobData.Metrics[metric]) == 0 {
+				delete(jobData.Metrics, metric)
 			}
 		}
 	}
@@ -407,7 +407,7 @@ func (ccms *InternalMetricStore) LoadScopedStats(
 	queries, assignedScope, err := buildQueries(job, metrics, scopes, 0)
 	if err != nil {
 		cclog.Errorf("Error while building queries for jobId %d, Metrics %v, Scopes %v: %s", job.JobID, metrics, scopes, err.Error())
-		return nil, err
+		return schema.ScopedJobStats{}, err
 	}
 
 	req := APIQueryRequest{
@@ -422,11 +422,11 @@ func (ccms *InternalMetricStore) LoadScopedStats(
 	resBody, err := FetchData(req)
 	if err != nil {
 		cclog.Errorf("Error while fetching data : %s", err.Error())
-		return nil, err
+		return schema.ScopedJobStats{}, err
 	}
 
 	var errors []string
-	scopedJobStats := make(schema.ScopedJobStats)
+	scopedJobStats := schema.ScopedJobStats{Metrics: make(map[string]schema.ScopedMetricStats)}
 
 	for i, row := range resBody.Results {
 		if len(row) == 0 {
@@ -437,12 +437,12 @@ func (ccms *InternalMetricStore) LoadScopedStats(
 		metric := query.Metric
 		scope := assignedScope[i]
 
-		if _, ok := scopedJobStats[metric]; !ok {
-			scopedJobStats[metric] = make(map[schema.MetricScope][]*schema.ScopedStats)
+		if _, ok := scopedJobStats.Metrics[metric]; !ok {
+			scopedJobStats.Metrics[metric] = make(schema.ScopedMetricStats)
 		}
 
-		if _, ok := scopedJobStats[metric][scope]; !ok {
-			scopedJobStats[metric][scope] = make([]*schema.ScopedStats, 0)
+		if _, ok := scopedJobStats.Metrics[metric][scope]; !ok {
+			scopedJobStats.Metrics[metric][scope] = make([]*schema.ScopedStats, 0)
 		}
 
 		for ndx, res := range row {
@@ -456,7 +456,7 @@ func (ccms *InternalMetricStore) LoadScopedStats(
 
 			SanitizeStats(&res.Avg, &res.Min, &res.Max)
 
-			scopedJobStats[metric][scope] = append(scopedJobStats[metric][scope], &schema.ScopedStats{
+			scopedJobStats.Metrics[metric][scope] = append(scopedJobStats.Metrics[metric][scope], &schema.ScopedStats{
 				Hostname: query.Hostname,
 				ID:       id,
 				Data: &schema.MetricStatistics{
@@ -467,11 +467,11 @@ func (ccms *InternalMetricStore) LoadScopedStats(
 			})
 		}
 
-		// So that one can later check len(scopedJobStats[metric][scope]): Remove from map if empty
-		if len(scopedJobStats[metric][scope]) == 0 {
-			delete(scopedJobStats[metric], scope)
-			if len(scopedJobStats[metric]) == 0 {
-				delete(scopedJobStats, metric)
+		// So that one can later check len(scopedJobStats.Metrics[metric][scope]): Remove from map if empty
+		if len(scopedJobStats.Metrics[metric][scope]) == 0 {
+			delete(scopedJobStats.Metrics[metric], scope)
+			if len(scopedJobStats.Metrics[metric]) == 0 {
+				delete(scopedJobStats.Metrics, metric)
 			}
 		}
 	}
@@ -695,14 +695,14 @@ func (ccms *InternalMetricStore) LoadNodeListData(
 		// Init Nested Map Data Structures If Not Found
 		hostData, ok := data[query.Hostname]
 		if !ok {
-			hostData = make(schema.JobData)
+			hostData = schema.JobData{Metrics: make(map[string]schema.ScopedMetrics)}
 			data[query.Hostname] = hostData
 		}
 
-		metricData, ok := hostData[metric]
+		metricData, ok := hostData.Metrics[metric]
 		if !ok {
-			metricData = make(map[schema.MetricScope]*schema.JobMetric)
-			data[query.Hostname][metric] = metricData
+			metricData = make(schema.ScopedMetrics)
+			data[query.Hostname].Metrics[metric] = metricData
 		}
 
 		scopeData, ok := metricData[scope]
@@ -712,7 +712,7 @@ func (ccms *InternalMetricStore) LoadNodeListData(
 				Timestep: res,
 				Series:   make([]schema.Series, 0),
 			}
-			data[query.Hostname][metric][scope] = scopeData
+			data[query.Hostname].Metrics[metric][scope] = scopeData
 		}
 
 		for ndx, res := range row {
diff --git a/tools/archive-manager/import_test.go b/tools/archive-manager/import_test.go
index db8940c2..4273f489 100644
--- a/tools/archive-manager/import_test.go
+++ b/tools/archive-manager/import_test.go
@@ -231,9 +231,9 @@ func TestImportDataIntegrity(t *testing.T) {
 		}
 
 		// Verify metric data exists
-		if len(srcData) != len(dstData) {
+		if len(srcData.Metrics) != len(dstData.Metrics) {
 			t.Errorf("Metric count mismatch for job %d: expected %d, got %d",
-				srcJob.Meta.JobID, len(srcData), len(dstData))
+				srcJob.Meta.JobID, len(srcData.Metrics), len(dstData.Metrics))
 		}
 
 		verifiedJobs++