Merge pull request #452 from ClusterCockpit/dev

Dev

.gitignore

@@ -13,7 +13,7 @@
 /var/checkpoints*
 
 migrateTimestamps.pl
-test_ccms_write_api.sh
+test_ccms_write_api*
 
 /web/frontend/public/build
 /web/frontend/node_modules

AGENTS.md

@@ -0,0 +1,26 @@
+# ClusterCockpit Backend - Agent Guidelines
+
+## Build/Test Commands
+
+- Build: `make` or `go build ./cmd/cc-backend`
+- Run all tests: `make test` (runs: `go clean -testcache && go build ./... && go vet ./... && go test ./...`)
+- Run single test: `go test -run TestName ./path/to/package`
+- Run single test file: `go test ./path/to/package -run TestName`
+- Frontend build: `cd web/frontend && npm install && npm run build`
+- Generate GraphQL: `make graphql` (uses gqlgen)
+- Generate Swagger: `make swagger` (uses swaggo/swag)
+
+## Code Style
+
+- **Formatting**: Use `gofumpt` for all Go files (strict requirement)
+- **Copyright header**: All files must include copyright header (see existing files)
+- **Package docs**: Document packages with comprehensive package-level comments explaining purpose, usage, configuration
+- **Imports**: Standard library first, then external packages, then internal packages (grouped with blank lines)
+- **Naming**: Use camelCase for private, PascalCase for exported; descriptive names (e.g., `JobRepository`, `handleError`)
+- **Error handling**: Return errors, don't panic; use custom error types where appropriate; log with cclog package
+- **Logging**: Use `cclog` package (e.g., `cclog.Errorf()`, `cclog.Warnf()`, `cclog.Debugf()`)
+- **Testing**: Use standard `testing` package; use `testify/assert` for assertions; name tests `TestFunctionName`
+- **Comments**: Document all exported functions/types with godoc-style comments
+- **Structs**: Document fields with inline comments, especially for complex configurations
+- **HTTP handlers**: Return proper status codes; use `handleError()` helper for consistent error responses
+- **JSON**: Use struct tags for JSON marshaling; `DisallowUnknownFields()` for strict decoding

Makefile

@@ -50,12 +50,12 @@ frontend:
 
 swagger:
 	$(info ===>  GENERATE swagger)
-	@go run github.com/swaggo/swag/cmd/swag init  --parseDependency -d ./internal/api -g rest.go -o ./api
+	@go tool github.com/swaggo/swag/cmd/swag init  --parseDependency -d ./internal/api -g rest.go -o ./api
 	@mv ./api/docs.go ./internal/api/docs.go
 
 graphql:
 	$(info ===>  GENERATE graphql)
-	@go run github.com/99designs/gqlgen
+	@go tool github.com/99designs/gqlgen
 
 clean:
 	$(info ===>  CLEAN)

api/schema.graphqls

@@ -164,6 +164,13 @@ type JobMetricWithName {
   metric: JobMetric!
 }
 
+type ClusterMetricWithName {
+  name: String!
+  unit: Unit
+  timestep: Int!
+  data: [NullableFloat!]!
+}
+
 type JobMetric {
   unit: Unit
   timestep: Int!
@@ -267,6 +274,11 @@ type NodeMetrics {
   metrics: [JobMetricWithName!]!
 }
 
+type ClusterMetrics {
+  nodeCount: Int!
+  metrics: [ClusterMetricWithName!]!
+}
+
 type NodesResultList {
   items: [NodeMetrics!]!
   offset: Int
@@ -385,6 +397,13 @@ type Query {
     page: PageRequest
     resolution: Int
   ): NodesResultList!
+
+  clusterMetrics(
+    cluster: String!
+    metrics: [String!]
+    from: Time!
+    to: Time!
+  ): ClusterMetrics!
 }
 
 type Mutation {

cmd/cc-backend/cli.go

@@ -33,6 +33,6 @@ func cliInit() {
 	flag.StringVar(&flagDelUser, "del-user", "", "Remove a existing user. Argument format: <username>")
 	flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
 	flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
-	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info (default), warn, err, crit]`")
+	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info , warn (default), err, crit]`")
 	flag.Parse()
 }

cmd/cc-backend/main.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/tagger"
 	"github.com/ClusterCockpit/cc-backend/internal/taskmanager"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
+	"github.com/ClusterCockpit/cc-backend/pkg/nats"
 	"github.com/ClusterCockpit/cc-backend/web"
 	ccconf "github.com/ClusterCockpit/cc-lib/ccConfig"
 	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
@@ -262,11 +263,18 @@ func generateJWT(authHandle *auth.Authentication, username string) error {
 		return fmt.Errorf("generating JWT for user '%s': %w", user.Username, err)
 	}
 
-	cclog.Infof("JWT: Successfully generated JWT for user '%s': %s", user.Username, jwt)
+	cclog.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt)
 	return nil
 }
 
 func initSubsystems() error {
+	// Initialize nats client
+	natsConfig := ccconf.GetPackageConfig("nats")
+	if err := nats.Init(natsConfig); err != nil {
+		cclog.Warnf("initializing (optional) nats client: %s", err.Error())
+	}
+	nats.Connect()
+
 	// Initialize job archive
 	archiveCfg := ccconf.GetPackageConfig("archive")
 	if archiveCfg == nil {

cmd/cc-backend/server.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
 	"github.com/ClusterCockpit/cc-backend/internal/memorystore"
 	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
+	"github.com/ClusterCockpit/cc-backend/pkg/nats"
 	"github.com/ClusterCockpit/cc-backend/web"
 	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
 	"github.com/ClusterCockpit/cc-lib/runtimeEnv"
@@ -363,6 +364,11 @@ func (s *Server) Shutdown(ctx context.Context) {
 	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()
 
+	nc := nats.GetClient()
+	if nc != nil {
+		nc.Close()
+	}
+
 	// First shut down the server gracefully (waiting for all ongoing requests)
 	if err := s.server.Shutdown(shutdownCtx); err != nil {
 		cclog.Errorf("Server shutdown error: %v", err)

configs/config-demo.json

@@ -29,6 +29,11 @@
       "max-age": "2000h"
     }
   },
+  "nats": {
+    "address": "nats://0.0.0.0:4222",
+    "username": "root",
+    "password": "root"
+  },
   "clusters": [
     {
       "name": "fritz",
@@ -86,6 +91,16 @@
       "interval": "1h",
       "directory": "./var/archive"
     },
-    "retention-in-memory": "48h"
+    "retention-in-memory": "48h",
+    "subscriptions": [
+      {
+        "subscribe-to": "hpc-nats",
+        "cluster-tag": "fritz"
+      },
+      {
+        "subscribe-to": "hpc-nats",
+        "cluster-tag": "alex"
+      }
+    ]
   }
 }

configs/config.json

@@ -9,8 +9,12 @@
     "apiAllowedIPs": ["*"],
     "short-running-jobs-duration": 300,
     "resampling": {
-      "trigger": 30,
+      "minimumPoints": 600,
-      "resolutions": [600, 300, 120, 60]
+      "trigger": 180,
+      "resolutions": [
+        240,
+        60
+      ]
     }
   },
   "cron": {

go.mod

@@ -4,6 +4,11 @@ go 1.24.0
 
 toolchain go1.24.1
 
+tool (
+	github.com/99designs/gqlgen
+	github.com/swaggo/swag/cmd/swag
+)
+
 require (
 	github.com/99designs/gqlgen v0.17.84
 	github.com/ClusterCockpit/cc-lib v1.0.0

internal/api/job.go

@@ -253,7 +253,7 @@ func (api *RestAPI) getCompleteJobByID(rw http.ResponseWriter, r *http.Request)
 			return
 		}
 
-		job, err = api.JobRepository.FindById(r.Context(), id) // Get Job from Repo by ID
+		job, err = api.JobRepository.FindByID(r.Context(), id) // Get Job from Repo by ID
 	} else {
 		handleError(fmt.Errorf("the parameter 'id' is required"), http.StatusBadRequest, rw)
 		return
@@ -346,7 +346,7 @@ func (api *RestAPI) getJobByID(rw http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		job, err = api.JobRepository.FindById(r.Context(), id)
+		job, err = api.JobRepository.FindByID(r.Context(), id)
 	} else {
 		handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw)
 		return
@@ -445,7 +445,7 @@ func (api *RestAPI) editMeta(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	job, err := api.JobRepository.FindById(r.Context(), id)
+	job, err := api.JobRepository.FindByID(r.Context(), id)
 	if err != nil {
 		handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw)
 		return
@@ -493,7 +493,7 @@ func (api *RestAPI) tagJob(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	job, err := api.JobRepository.FindById(r.Context(), id)
+	job, err := api.JobRepository.FindByID(r.Context(), id)
 	if err != nil {
 		handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw)
 		return
@@ -557,7 +557,7 @@ func (api *RestAPI) removeTagJob(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	job, err := api.JobRepository.FindById(r.Context(), id)
+	job, err := api.JobRepository.FindByID(r.Context(), id)
 	if err != nil {
 		handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw)
 		return
@@ -796,7 +796,7 @@ func (api *RestAPI) deleteJobByID(rw http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		err = api.JobRepository.DeleteJobById(id)
+		err = api.JobRepository.DeleteJobByID(id)
 	} else {
 		handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw)
 		return
@@ -852,7 +852,7 @@ func (api *RestAPI) deleteJobByRequest(rw http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	err = api.JobRepository.DeleteJobById(*job.ID)
+	err = api.JobRepository.DeleteJobByID(*job.ID)
 	if err != nil {
 		handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw)
 		return

internal/api/nats.go

@@ -0,0 +1,231 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package api
+
+import (
+	"bytes"
+	"database/sql"
+	"encoding/json"
+	"sync"
+	"time"
+
+	"github.com/ClusterCockpit/cc-backend/internal/archiver"
+	"github.com/ClusterCockpit/cc-backend/internal/config"
+	"github.com/ClusterCockpit/cc-backend/internal/importer"
+	"github.com/ClusterCockpit/cc-backend/internal/repository"
+	"github.com/ClusterCockpit/cc-backend/pkg/nats"
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/ClusterCockpit/cc-lib/schema"
+)
+
+// NatsAPI provides NATS subscription-based handlers for Job and Node operations.
+// It mirrors the functionality of the REST API but uses NATS messaging.
+type NatsAPI struct {
+	JobRepository *repository.JobRepository
+	// RepositoryMutex protects job creation operations from race conditions
+	// when checking for duplicate jobs during startJob calls.
+	RepositoryMutex sync.Mutex
+}
+
+// NewNatsAPI creates a new NatsAPI instance with default dependencies.
+func NewNatsAPI() *NatsAPI {
+	return &NatsAPI{
+		JobRepository: repository.GetJobRepository(),
+	}
+}
+
+// StartSubscriptions registers all NATS subscriptions for Job and Node APIs.
+// Returns an error if the NATS client is not available or subscription fails.
+func (api *NatsAPI) StartSubscriptions() error {
+	client := nats.GetClient()
+	if client == nil {
+		cclog.Warn("NATS client not available, skipping API subscriptions")
+		return nil
+	}
+
+	if config.Keys.APISubjects != nil {
+
+		s := config.Keys.APISubjects
+
+		if err := client.Subscribe(s.SubjectJobStart, api.handleStartJob); err != nil {
+			return err
+		}
+
+		if err := client.Subscribe(s.SubjectJobStop, api.handleStopJob); err != nil {
+			return err
+		}
+
+		if err := client.Subscribe(s.SubjectNodeState, api.handleNodeState); err != nil {
+			return err
+		}
+
+		cclog.Info("NATS API subscriptions started")
+	}
+	return nil
+}
+
+// handleStartJob processes job start messages received via NATS.
+// Expected JSON payload follows the schema.Job structure.
+func (api *NatsAPI) handleStartJob(subject string, data []byte) {
+	req := schema.Job{
+		Shared:           "none",
+		MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
+	}
+
+	dec := json.NewDecoder(bytes.NewReader(data))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&req); err != nil {
+		cclog.Errorf("NATS %s: parsing request failed: %v", subject, err)
+		return
+	}
+
+	cclog.Debugf("NATS %s: %s", subject, req.GoString())
+	req.State = schema.JobStateRunning
+
+	if err := importer.SanityChecks(&req); err != nil {
+		cclog.Errorf("NATS %s: sanity check failed: %v", subject, err)
+		return
+	}
+
+	var unlockOnce sync.Once
+	api.RepositoryMutex.Lock()
+	defer unlockOnce.Do(api.RepositoryMutex.Unlock)
+
+	jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil)
+	if err != nil && err != sql.ErrNoRows {
+		cclog.Errorf("NATS %s: checking for duplicate failed: %v", subject, err)
+		return
+	}
+	if err == nil {
+		for _, job := range jobs {
+			if (req.StartTime - job.StartTime) < secondsPerDay {
+				cclog.Errorf("NATS %s: job with jobId %d, cluster %s already exists (dbid: %d)",
+					subject, req.JobID, req.Cluster, job.ID)
+				return
+			}
+		}
+	}
+
+	id, err := api.JobRepository.Start(&req)
+	if err != nil {
+		cclog.Errorf("NATS %s: insert into database failed: %v", subject, err)
+		return
+	}
+	unlockOnce.Do(api.RepositoryMutex.Unlock)
+
+	for _, tag := range req.Tags {
+		if _, err := api.JobRepository.AddTagOrCreate(nil, id, tag.Type, tag.Name, tag.Scope); err != nil {
+			cclog.Errorf("NATS %s: adding tag to new job %d failed: %v", subject, id, err)
+			return
+		}
+	}
+
+	cclog.Infof("NATS: new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d",
+		id, req.Cluster, req.JobID, req.User, req.StartTime)
+}
+
+// handleStopJob processes job stop messages received via NATS.
+// Expected JSON payload follows the StopJobAPIRequest structure.
+func (api *NatsAPI) handleStopJob(subject string, data []byte) {
+	var req StopJobAPIRequest
+
+	dec := json.NewDecoder(bytes.NewReader(data))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&req); err != nil {
+		cclog.Errorf("NATS %s: parsing request failed: %v", subject, err)
+		return
+	}
+
+	if req.JobID == nil {
+		cclog.Errorf("NATS %s: the field 'jobId' is required", subject)
+		return
+	}
+
+	job, err := api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime)
+	if err != nil {
+		cachedJob, cachedErr := api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime)
+		if cachedErr != nil {
+			cclog.Errorf("NATS %s: finding job failed: %v (cached lookup also failed: %v)",
+				subject, err, cachedErr)
+			return
+		}
+		job = cachedJob
+	}
+
+	if job.State != schema.JobStateRunning {
+		cclog.Errorf("NATS %s: jobId %d (id %d) on %s: job has already been stopped (state is: %s)",
+			subject, job.JobID, job.ID, job.Cluster, job.State)
+		return
+	}
+
+	if job.StartTime > req.StopTime {
+		cclog.Errorf("NATS %s: jobId %d (id %d) on %s: stopTime %d must be >= startTime %d",
+			subject, job.JobID, job.ID, job.Cluster, req.StopTime, job.StartTime)
+		return
+	}
+
+	if req.State != "" && !req.State.Valid() {
+		cclog.Errorf("NATS %s: jobId %d (id %d) on %s: invalid job state: %#v",
+			subject, job.JobID, job.ID, job.Cluster, req.State)
+		return
+	} else if req.State == "" {
+		req.State = schema.JobStateCompleted
+	}
+
+	job.Duration = int32(req.StopTime - job.StartTime)
+	job.State = req.State
+	api.JobRepository.Mutex.Lock()
+	defer api.JobRepository.Mutex.Unlock()
+
+	if err := api.JobRepository.Stop(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil {
+		if err := api.JobRepository.StopCached(*job.ID, job.Duration, job.State, job.MonitoringStatus); err != nil {
+			cclog.Errorf("NATS %s: jobId %d (id %d) on %s: marking job as '%s' failed: %v",
+				subject, job.JobID, job.ID, job.Cluster, job.State, err)
+			return
+		}
+	}
+
+	cclog.Infof("NATS: archiving job (dbid: %d): cluster=%s, jobId=%d, user=%s, startTime=%d, duration=%d, state=%s",
+		job.ID, job.Cluster, job.JobID, job.User, job.StartTime, job.Duration, job.State)
+
+	if job.MonitoringStatus == schema.MonitoringStatusDisabled {
+		return
+	}
+
+	archiver.TriggerArchiving(job)
+}
+
+// handleNodeState processes node state update messages received via NATS.
+// Expected JSON payload follows the UpdateNodeStatesRequest structure.
+func (api *NatsAPI) handleNodeState(subject string, data []byte) {
+	var req UpdateNodeStatesRequest
+
+	dec := json.NewDecoder(bytes.NewReader(data))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&req); err != nil {
+		cclog.Errorf("NATS %s: parsing request failed: %v", subject, err)
+		return
+	}
+
+	repo := repository.GetNodeRepository()
+
+	for _, node := range req.Nodes {
+		state := determineState(node.States)
+		nodeState := schema.NodeStateDB{
+			TimeStamp:       time.Now().Unix(),
+			NodeState:       state,
+			CpusAllocated:   node.CpusAllocated,
+			MemoryAllocated: node.MemoryAllocated,
+			GpusAllocated:   node.GpusAllocated,
+			HealthState:     schema.MonitoringStateFull,
+			JobsRunning:     node.JobsRunning,
+		}
+
+		repo.UpdateNodeState(node.Hostname, req.Cluster, &nodeState)
+	}
+
+	cclog.Debugf("NATS %s: updated %d node states for cluster %s", subject, len(req.Nodes), req.Cluster)
+}

internal/config/config.go

@@ -22,6 +22,8 @@ type ProgramConfig struct {
 	// Addresses from which secured admin API endpoints can be reached, can be wildcard "*"
 	APIAllowedIPs []string `json:"apiAllowedIPs"`
 
+	APISubjects *NATSConfig `json:"apiSubjects"`
+
 	// Drop root permissions once .env was read and the port was taken.
 	User  string `json:"user"`
 	Group string `json:"group"`
@@ -87,6 +89,12 @@ type ResampleConfig struct {
 	Trigger int `json:"trigger"`
 }
 
+type NATSConfig struct {
+	SubjectJobStart  string `json:"subjectJobStart"`
+	SubjectJobStop   string `json:"subjectJobStop"`
+	SubjectNodeState string `json:"subjectNodeState"`
+}
+
 type IntRange struct {
 	From int `json:"from"`
 	To   int `json:"to"`

internal/graph/model/models_gen.go

@@ -13,6 +13,18 @@ import (
 	"github.com/ClusterCockpit/cc-lib/schema"
 )
 
+type ClusterMetricWithName struct {
+	Name     string         `json:"name"`
+	Unit     *schema.Unit   `json:"unit,omitempty"`
+	Timestep int            `json:"timestep"`
+	Data     []schema.Float `json:"data"`
+}
+
+type ClusterMetrics struct {
+	NodeCount int                      `json:"nodeCount"`
+	Metrics   []*ClusterMetricWithName `json:"metrics"`
+}
+
 type Count struct {
 	Name  string `json:"name"`
 	Count int    `json:"count"`

internal/graph/schema.resolvers.go

@@ -1,13 +1,15 @@
 package graph
 
-// This file will be automatically regenerated based on the schema, any resolver implementations
+// This file will be automatically regenerated based on the schema, any resolver
+// implementations
 // will be copied through when generating and any unknown code will be moved to the end.
-// Code generated by github.com/99designs/gqlgen version v0.17.81
+// Code generated by github.com/99designs/gqlgen version v0.17.84
 
 import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"regexp"
 	"slices"
 	"strconv"
@@ -376,7 +378,7 @@ func (r *queryResolver) Node(ctx context.Context, id string) (*schema.Node, erro
 		cclog.Warn("Error while parsing job id")
 		return nil, err
 	}
-	return repo.GetNodeById(numericId, false)
+	return repo.GetNodeByID(numericId, false)
 }
 
 // Nodes is the resolver for the nodes field.
@@ -442,7 +444,7 @@ func (r *queryResolver) Job(ctx context.Context, id string) (*schema.Job, error)
 		return nil, err
 	}
 
-	job, err := r.Repo.FindById(ctx, numericId)
+	job, err := r.Repo.FindByID(ctx, numericId)
 	if err != nil {
 		cclog.Warn("Error while finding job by id")
 		return nil, err
@@ -804,140 +806,24 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub
 		return nil, errors.New("you need to be administrator or support staff for this query")
 	}
 
+	nodeRepo := repository.GetNodeRepository()
+	nodes, stateMap, countNodes, hasNextPage, nerr := nodeRepo.GetNodesForList(ctx, cluster, subCluster, stateFilter, nodeFilter, page)
+	if nerr != nil {
+		return nil, errors.New("Could not retrieve node list required for resolving NodeMetricsList")
+	}
+
 	if metrics == nil {
 		for _, mc := range archive.GetCluster(cluster).MetricConfig {
 			metrics = append(metrics, mc.Name)
 		}
 	}
 
-	// Build Filters
-	queryFilters := make([]*model.NodeFilter, 0)
-	if cluster != "" {
-		queryFilters = append(queryFilters, &model.NodeFilter{Cluster: &model.StringInput{Eq: &cluster}})
-	}
-	if subCluster != "" {
-		queryFilters = append(queryFilters, &model.NodeFilter{Subcluster: &model.StringInput{Eq: &subCluster}})
-	}
-	if nodeFilter != "" && stateFilter != "notindb" {
-		queryFilters = append(queryFilters, &model.NodeFilter{Hostname: &model.StringInput{Contains: &nodeFilter}})
-	}
-	if stateFilter != "all" && stateFilter != "notindb" {
-		var queryState schema.SchedulerState = schema.SchedulerState(stateFilter)
-		queryFilters = append(queryFilters, &model.NodeFilter{SchedulerState: &queryState})
-	}
-	// if healthFilter != "all" {
-	// 	filters = append(filters, &model.NodeFilter{HealthState: &healthFilter})
-	// }
-
-	// Special Case: Disable Paging for missing nodes filter, save IPP for later
-	var backupItems int
-	if stateFilter == "notindb" {
-		backupItems = page.ItemsPerPage
-		page.ItemsPerPage = -1
-	}
-
-	// Query Nodes From DB
-	nodeRepo := repository.GetNodeRepository()
-	rawNodes, serr := nodeRepo.QueryNodes(ctx, queryFilters, page, nil) // Order not Used
-	if serr != nil {
-		cclog.Warn("error while loading node database data (Resolver.NodeMetricsList)")
-		return nil, serr
-	}
-
-	// Intermediate Node Result Info
-	nodes := make([]string, 0)
-	stateMap := make(map[string]string)
-	for _, node := range rawNodes {
-		nodes = append(nodes, node.Hostname)
-		stateMap[node.Hostname] = string(node.NodeState)
-	}
-
-	// Setup Vars
-	var countNodes int
-	var cerr error
-	var hasNextPage bool
-
-	// Special Case: Find Nodes not in DB node table but in metricStore only
-	if stateFilter == "notindb" {
-		// Reapply Original Paging
-		page.ItemsPerPage = backupItems
-		// Get Nodes From Topology
-		var topoNodes []string
-		if subCluster != "" {
-			scNodes := archive.NodeLists[cluster][subCluster]
-			topoNodes = scNodes.PrintList()
-		} else {
-			subClusterNodeLists := archive.NodeLists[cluster]
-			for _, nodeList := range subClusterNodeLists {
-				topoNodes = append(topoNodes, nodeList.PrintList()...)
-			}
-		}
-		// Compare to all nodes from cluster/subcluster in DB
-		var missingNodes []string
-		for _, scanNode := range topoNodes {
-			if !slices.Contains(nodes, scanNode) {
-				missingNodes = append(missingNodes, scanNode)
-			}
-		}
-		// Filter nodes by name
-		if nodeFilter != "" {
-			filteredNodesByName := []string{}
-			for _, missingNode := range missingNodes {
-				if strings.Contains(missingNode, nodeFilter) {
-					filteredNodesByName = append(filteredNodesByName, missingNode)
-				}
-			}
-			missingNodes = filteredNodesByName
-		}
-		// Sort Missing Nodes Alphanumerically
-		slices.Sort(missingNodes)
-		// Total Missing
-		countNodes = len(missingNodes)
-		// Apply paging
-		if countNodes > page.ItemsPerPage {
-			start := (page.Page - 1) * page.ItemsPerPage
-			end := start + page.ItemsPerPage
-			if end > countNodes {
-				end = countNodes
-				hasNextPage = false
-			} else {
-				hasNextPage = true
-			}
-			nodes = missingNodes[start:end]
-		} else {
-			nodes = missingNodes
-		}
-
-	} else {
-		// DB Nodes: Count and Find Next Page
-		countNodes, cerr = nodeRepo.CountNodes(ctx, queryFilters)
-		if cerr != nil {
-			cclog.Warn("error while counting node database data (Resolver.NodeMetricsList)")
-			return nil, cerr
-		}
-
-		// Example Page 4 @ 10 IpP : Does item 41 exist?
-		// Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 exists.
-		nextPage := &model.PageRequest{
-			ItemsPerPage: 1,
-			Page:         ((page.Page * page.ItemsPerPage) + 1),
-		}
-		nextNodes, err := nodeRepo.QueryNodes(ctx, queryFilters, nextPage, nil) // Order not Used
-		if err != nil {
-			cclog.Warn("Error while querying next nodes")
-			return nil, err
-		}
-		hasNextPage = len(nextNodes) == 1
-	}
-
-	// Load Metric Data For Specified Nodes Only
 	data, err := metricDataDispatcher.LoadNodeListData(cluster, subCluster, nodes, metrics, scopes, *resolution, from, to, ctx)
 	if err != nil {
 		cclog.Warn("error while loading node data (Resolver.NodeMetricsList")
 		return nil, err
 	}
 
-	// Build Result
 	nodeMetricsList := make([]*model.NodeMetrics, 0, len(data))
 	for hostname, metrics := range data {
 		host := &model.NodeMetrics{
@@ -963,7 +849,6 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub
 		nodeMetricsList = append(nodeMetricsList, host)
 	}
 
-	// Final Return
 	nodeMetricsListResult := &model.NodesResultList{
 		Items:       nodeMetricsList,
 		TotalNodes:  &countNodes,
@@ -973,6 +858,85 @@ func (r *queryResolver) NodeMetricsList(ctx context.Context, cluster string, sub
 	return nodeMetricsListResult, nil
 }
 
+// ClusterMetrics is the resolver for the clusterMetrics field.
+func (r *queryResolver) ClusterMetrics(ctx context.Context, cluster string, metrics []string, from time.Time, to time.Time) (*model.ClusterMetrics, error) {
+	user := repository.GetUserFromContext(ctx)
+	if user != nil && !user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}) {
+		return nil, errors.New("you need to be administrator or support staff for this query")
+	}
+
+	if metrics == nil {
+		for _, mc := range archive.GetCluster(cluster).MetricConfig {
+			metrics = append(metrics, mc.Name)
+		}
+	}
+
+	// 'nodes' == nil -> Defaults to all nodes of cluster for existing query workflow
+	scopes := []schema.MetricScope{"node"}
+	data, err := metricDataDispatcher.LoadNodeData(cluster, metrics, nil, scopes, from, to, ctx)
+	if err != nil {
+		cclog.Warn("error while loading node data")
+		return nil, err
+	}
+
+	clusterMetricData := make([]*model.ClusterMetricWithName, 0)
+	clusterMetrics := model.ClusterMetrics{NodeCount: 0, Metrics: clusterMetricData}
+
+	collectorTimestep := make(map[string]int)
+	collectorUnit := make(map[string]schema.Unit)
+	collectorData := make(map[string][]schema.Float)
+
+	for _, metrics := range data {
+		clusterMetrics.NodeCount += 1
+		for metric, scopedMetrics := range metrics {
+			_, ok := collectorData[metric]
+			if !ok {
+				collectorData[metric] = make([]schema.Float, 0)
+				for _, scopedMetric := range scopedMetrics {
+					// Collect Info
+					collectorTimestep[metric] = scopedMetric.Timestep
+					collectorUnit[metric] = scopedMetric.Unit
+					// Collect Initial Data
+					for _, ser := range scopedMetric.Series {
+						for _, val := range ser.Data {
+							collectorData[metric] = append(collectorData[metric], val)
+						}
+					}
+				}
+			} else {
+				// Sum up values by index
+				for _, scopedMetric := range scopedMetrics {
+					// For This Purpose (Cluster_Wide-Sum of Node Metrics) OK
+					for _, ser := range scopedMetric.Series {
+						for i, val := range ser.Data {
+							collectorData[metric][i] += val
+						}
+					}
+				}
+			}
+		}
+	}
+
+	for metricName, data := range collectorData {
+		cu := collectorUnit[metricName]
+		roundedData := make([]schema.Float, 0)
+		for _, val := range data {
+			roundedData = append(roundedData, schema.Float((math.Round(float64(val)*100.0) / 100.0)))
+		}
+
+		cm := model.ClusterMetricWithName{
+			Name:     metricName,
+			Unit:     &cu,
+			Timestep: collectorTimestep[metricName],
+			Data:     roundedData,
+		}
+
+		clusterMetrics.Metrics = append(clusterMetrics.Metrics, &cm)
+	}
+
+	return &clusterMetrics, nil
+}
+
 // NumberOfNodes is the resolver for the numberOfNodes field.
 func (r *subClusterResolver) NumberOfNodes(ctx context.Context, obj *schema.SubCluster) (int, error) {
 	nodeList, err := archive.ParseNodeList(obj.Nodes)
@@ -1003,10 +967,12 @@ func (r *Resolver) Query() generated.QueryResolver { return &queryResolver{r} }
 // SubCluster returns generated.SubClusterResolver implementation.
 func (r *Resolver) SubCluster() generated.SubClusterResolver { return &subClusterResolver{r} }
 
-type clusterResolver struct{ *Resolver }
+type (
-type jobResolver struct{ *Resolver }
+	clusterResolver     struct{ *Resolver }
-type metricValueResolver struct{ *Resolver }
+	jobResolver         struct{ *Resolver }
-type mutationResolver struct{ *Resolver }
+	metricValueResolver struct{ *Resolver }
-type nodeResolver struct{ *Resolver }
+	mutationResolver    struct{ *Resolver }
-type queryResolver struct{ *Resolver }
+	nodeResolver        struct{ *Resolver }
-type subClusterResolver struct{ *Resolver }
+	queryResolver       struct{ *Resolver }
+	subClusterResolver  struct{ *Resolver }
+)

internal/graph/util.go

@@ -2,12 +2,14 @@
 // All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package graph
 
 import (
 	"context"
 	"fmt"
 	"math"
+	"slices"
 
 	"github.com/99designs/gqlgen/graphql"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
@@ -185,11 +187,5 @@ func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobF
 func requireField(ctx context.Context, name string) bool {
 	fields := graphql.CollectAllFields(ctx)
 
-	for _, f := range fields {
+	return slices.Contains(fields, name)
-		if f == name {
-			return true
-		}
-	}
-
-	return false
 }

internal/importer/initDB.go

@@ -111,18 +111,22 @@ func InitDB() error {
 			continue
 		}
 
-		id, err := r.TransactionAddNamed(t,
+		id, jobErr := r.TransactionAddNamed(t,
 			repository.NamedJobInsert, jobMeta)
-		if err != nil {
+		if jobErr != nil {
-			cclog.Errorf("repository initDB(): %v", err)
+			cclog.Errorf("repository initDB(): %v", jobErr)
 			errorOccured++
 			continue
 		}
 
+		// Job successfully inserted, increment counter
+		i += 1
+
 		for _, tag := range jobMeta.Tags {
 			tagstr := tag.Name + ":" + tag.Type
 			tagID, ok := tags[tagstr]
 			if !ok {
+				var err error
 				tagID, err = r.TransactionAdd(t,
 					addTagQuery,
 					tag.Name, tag.Type)
@@ -138,10 +142,6 @@ func InitDB() error {
 				setTagQuery,
 				id, tagID)
 		}
-
-		if err == nil {
-			i += 1
-		}
 	}
 
 	if errorOccured > 0 {

internal/memorystore/api.go

@@ -6,12 +6,18 @@
 package memorystore
 
 import (
+	"errors"
 	"math"
 
 	"github.com/ClusterCockpit/cc-lib/schema"
 	"github.com/ClusterCockpit/cc-lib/util"
 )
 
+var (
+	ErrInvalidTimeRange = errors.New("[METRICSTORE]> invalid time range: 'from' must be before 'to'")
+	ErrEmptyCluster     = errors.New("[METRICSTORE]> cluster name cannot be empty")
+)
+
 type APIMetricData struct {
 	Error      *string           `json:"error,omitempty"`
 	Data       schema.FloatArray `json:"data,omitempty"`
@@ -109,10 +115,14 @@ func (data *APIMetricData) PadDataWithNull(ms *MemoryStore, from, to int64, metr
 }
 
 func FetchData(req APIQueryRequest) (*APIQueryResponse, error) {
-	req.WithData = true
+	if req.From > req.To {
-	req.WithData = true
+		return nil, ErrInvalidTimeRange
-	req.WithData = true
+	}
+	if req.Cluster == "" && req.ForAllNodes != nil {
+		return nil, ErrEmptyCluster
+	}
 
+	req.WithData = true
 	ms := GetMemoryStore()
 
 	response := APIQueryResponse{

internal/memorystore/archive.go

@@ -32,17 +32,14 @@ func Archiving(wg *sync.WaitGroup, ctx context.Context) {
 			return
 		}
 
-		ticks := func() <-chan time.Time {
+		ticker := time.NewTicker(d)
-			if d <= 0 {
+		defer ticker.Stop()
-				return nil
+
-			}
-			return time.NewTicker(d).C
-		}()
 		for {
 			select {
 			case <-ctx.Done():
 				return
-			case <-ticks:
+			case <-ticker.C:
 				t := time.Now().Add(-d)
 				cclog.Infof("[METRICSTORE]> start archiving checkpoints (older than %s)...", t.Format(time.RFC3339))
 				n, err := ArchiveCheckpoints(Keys.Checkpoints.RootDir,
@@ -165,25 +162,33 @@ func archiveCheckpoints(dir string, archiveDir string, from int64, deleteInstead
 
 	n := 0
 	for _, checkpoint := range files {
-		filename := filepath.Join(dir, checkpoint)
+		// Use closure to ensure file is closed immediately after use,
-		r, err := os.Open(filename)
+		// avoiding file descriptor leak from defer in loop
+		err := func() error {
+			filename := filepath.Join(dir, checkpoint)
+			r, err := os.Open(filename)
+			if err != nil {
+				return err
+			}
+			defer r.Close()
+
+			w, err := zw.Create(checkpoint)
+			if err != nil {
+				return err
+			}
+
+			if _, err = io.Copy(w, r); err != nil {
+				return err
+			}
+
+			if err = os.Remove(filename); err != nil {
+				return err
+			}
+			return nil
+		}()
 		if err != nil {
 			return n, err
 		}
-		defer r.Close()
-
-		w, err := zw.Create(checkpoint)
-		if err != nil {
-			return n, err
-		}
-
-		if _, err = io.Copy(w, r); err != nil {
-			return n, err
-		}
-
-		if err = os.Remove(filename); err != nil {
-			return n, err
-		}
 		n += 1
 	}
 

internal/memorystore/avroCheckpoint.go

@@ -24,9 +24,8 @@ import (
 	"github.com/linkedin/goavro/v2"
 )
 
-var NumAvroWorkers int = 4
+var NumAvroWorkers int = DefaultAvroWorkers
 var startUp bool = true
-var ErrNoNewData error = errors.New("no data in the pool")
 
 func (as *AvroStore) ToCheckpoint(dir string, dumpAll bool) (int, error) {
 	levels := make([]*AvroLevel, 0)
@@ -464,19 +463,15 @@ func generateRecord(data map[string]schema.Float) map[string]any {
 }
 
 func correctKey(key string) string {
-	// Replace any invalid characters in the key
+	key = strings.ReplaceAll(key, "_", "_0x5F_")
-	// For example, replace spaces with underscores
+	key = strings.ReplaceAll(key, ":", "_0x3A_")
-	key = strings.ReplaceAll(key, ":", "___")
+	key = strings.ReplaceAll(key, ".", "_0x2E_")
-	key = strings.ReplaceAll(key, ".", "__")
-
 	return key
 }
 
 func ReplaceKey(key string) string {
-	// Replace any invalid characters in the key
+	key = strings.ReplaceAll(key, "_0x2E_", ".")
-	// For example, replace spaces with underscores
+	key = strings.ReplaceAll(key, "_0x3A_", ":")
-	key = strings.ReplaceAll(key, "___", ":")
+	key = strings.ReplaceAll(key, "_0x5F_", "_")
-	key = strings.ReplaceAll(key, "__", ".")
-
 	return key
 }

internal/memorystore/avroHelper.go

@@ -42,7 +42,7 @@ func DataStaging(wg *sync.WaitGroup, ctx context.Context) {
 				metricName := ""
 
 				for _, selectorName := range val.Selector {
-					metricName += selectorName + Delimiter
+					metricName += selectorName + SelectorDelimiter
 				}
 
 				metricName += val.MetricName
@@ -54,7 +54,7 @@ func DataStaging(wg *sync.WaitGroup, ctx context.Context) {
 				var selector []string
 				selector = append(selector, val.Cluster, val.Node, strconv.FormatInt(freq, 10))
 
-				if !testEq(oldSelector, selector) {
+				if !stringSlicesEqual(oldSelector, selector) {
 					// Get the Avro level for the metric
 					avroLevel = avroStore.root.findAvroLevelOrCreate(selector)
 
@@ -71,7 +71,7 @@ func DataStaging(wg *sync.WaitGroup, ctx context.Context) {
 	}()
 }
 
-func testEq(a, b []string) bool {
+func stringSlicesEqual(a, b []string) bool {
 	if len(a) != len(b) {
 		return false
 	}

internal/memorystore/avroStruct.go

@@ -13,12 +13,11 @@ import (
 
 var (
 	LineProtocolMessages = make(chan *AvroStruct)
-	Delimiter            = "ZZZZZ"
+	// SelectorDelimiter separates hierarchical selector components in metric names for Avro encoding
+	SelectorDelimiter = "_SEL_"
 )
 
-// CheckpointBufferMinutes should always be in minutes.
+var CheckpointBufferMinutes = DefaultCheckpointBufferMin
-// Its controls the amount of data to hold for given amount of time.
-var CheckpointBufferMinutes = 3
 
 type AvroStruct struct {
 	MetricName string
@@ -73,7 +72,7 @@ func (l *AvroLevel) findAvroLevelOrCreate(selector []string) *AvroLevel {
 		}
 	}
 
-	// The level does not exist, take write lock for unqiue access:
+	// The level does not exist, take write lock for unique access:
 	l.lock.Lock()
 	// While this thread waited for the write lock, another thread
 	// could have created the child node.

internal/memorystore/buffer.go

@@ -12,15 +12,12 @@ import (
 	"github.com/ClusterCockpit/cc-lib/schema"
 )
 
-// Default buffer capacity.
+// BufferCap is the default buffer capacity.
-// `buffer.data` will only ever grow up to it's capacity and a new link
+// buffer.data will only ever grow up to its capacity and a new link
 // in the buffer chain will be created if needed so that no copying
 // of data or reallocation needs to happen on writes.
-const (
+const BufferCap int = DefaultBufferCapacity
-	BufferCap int = 512
-)
 
-// So that we can reuse allocations
 var bufferPool sync.Pool = sync.Pool{
 	New: func() any {
 		return &buffer{
@@ -75,7 +72,6 @@ func (b *buffer) write(ts int64, value schema.Float) (*buffer, error) {
 		newbuf := newBuffer(ts, b.frequency)
 		newbuf.prev = b
 		b.next = newbuf
-		b.close()
 		b = newbuf
 		idx = 0
 	}
@@ -103,8 +99,6 @@ func (b *buffer) firstWrite() int64 {
 	return b.start + (b.frequency / 2)
 }
 
-func (b *buffer) close() {}
-
 // Return all known values from `from` to `to`. Gaps of information are represented as NaN.
 // Simple linear interpolation is done between the two neighboring cells if possible.
 // If values at the start or end are missing, instead of NaN values, the second and thrid
@@ -139,8 +133,6 @@ func (b *buffer) read(from, to int64, data []schema.Float) ([]schema.Float, int6
 			data[i] += schema.NaN
 		} else if t < b.start {
 			data[i] += schema.NaN
-			// } else if b.data[idx].IsNaN() {
-			// 	data[i] += interpolate(idx, b.data)
 		} else {
 			data[i] += b.data[idx]
 		}

internal/memorystore/checkpoint.go

@@ -28,15 +28,10 @@ import (
 	"github.com/linkedin/goavro/v2"
 )
 
-// File operation constants
 const (
-	// CheckpointFilePerms defines default permissions for checkpoint files
 	CheckpointFilePerms = 0o644
-	// CheckpointDirPerms defines default permissions for checkpoint directories
+	CheckpointDirPerms  = 0o755
-	CheckpointDirPerms = 0o755
+	GCTriggerInterval   = DefaultGCTriggerInterval
-	// GCTriggerInterval determines how often GC is forced during checkpoint loading
-	// GC is triggered every GCTriggerInterval*NumWorkers loaded hosts
-	GCTriggerInterval = 100
 )
 
 // Whenever changed, update MarshalJSON as well!
@@ -71,17 +66,14 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
 				return
 			}
 
-			ticks := func() <-chan time.Time {
+			ticker := time.NewTicker(d)
-				if d <= 0 {
+			defer ticker.Stop()
-					return nil
+
-				}
-				return time.NewTicker(d).C
-			}()
 			for {
 				select {
 				case <-ctx.Done():
 					return
-				case <-ticks:
+				case <-ticker.C:
 					cclog.Infof("[METRICSTORE]> start checkpointing (starting at %s)...", lastCheckpoint.Format(time.RFC3339))
 					now := time.Now()
 					n, err := ms.ToCheckpoint(Keys.Checkpoints.RootDir,
@@ -98,33 +90,23 @@ func Checkpointing(wg *sync.WaitGroup, ctx context.Context) {
 	} else {
 		go func() {
 			defer wg.Done()
-			d, _ := time.ParseDuration("1m")
 
 			select {
 			case <-ctx.Done():
 				return
 			case <-time.After(time.Duration(CheckpointBufferMinutes) * time.Minute):
-				// This is the first tick untill we collect the data for given minutes.
 				GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false)
-				// log.Printf("Checkpointing %d avro files", count)
-
 			}
 
-			ticks := func() <-chan time.Time {
+			ticker := time.NewTicker(DefaultAvroCheckpointInterval)
-				if d <= 0 {
+			defer ticker.Stop()
-					return nil
-				}
-				return time.NewTicker(d).C
-			}()
 
 			for {
 				select {
 				case <-ctx.Done():
 					return
-				case <-ticks:
+				case <-ticker.C:
-					// Regular ticks of 1 minute to write data.
 					GetAvroStore().ToCheckpoint(Keys.Checkpoints.RootDir, false)
-					// log.Printf("Checkpointing %d avro files", count)
 				}
 			}
 		}()
@@ -329,7 +311,7 @@ func (m *MemoryStore) FromCheckpoint(dir string, from int64, extension string) (
 				lvl := m.root.findLevelOrCreate(host[:], len(m.Metrics))
 				nn, err := lvl.fromCheckpoint(m, filepath.Join(dir, host[0], host[1]), from, extension)
 				if err != nil {
-					cclog.Fatalf("[METRICSTORE]> error while loading checkpoints: %s", err.Error())
+					cclog.Errorf("[METRICSTORE]> error while loading checkpoints for %s/%s: %s", host[0], host[1], err.Error())
 					atomic.AddInt32(&errs, 1)
 				}
 				atomic.AddInt32(&n, int32(nn))
@@ -506,8 +488,8 @@ func (l *Level) loadAvroFile(m *MemoryStore, f *os.File, from int64) error {
 	for key, floatArray := range metricsData {
 		metricName := ReplaceKey(key)
 
-		if strings.Contains(metricName, Delimiter) {
+		if strings.Contains(metricName, SelectorDelimiter) {
-			subString := strings.Split(metricName, Delimiter)
+			subString := strings.Split(metricName, SelectorDelimiter)
 
 			lvl := l
 
@@ -557,12 +539,10 @@ func (l *Level) createBuffer(m *MemoryStore, metricName string, floatArray schem
 		next:      nil,
 		archived:  true,
 	}
-	b.close()
 
 	minfo, ok := m.Metrics[metricName]
 	if !ok {
 		return nil
-		// return errors.New("Unkown metric: " + name)
 	}
 
 	prev := l.metrics[minfo.offset]
@@ -616,17 +596,15 @@ func (l *Level) loadFile(cf *CheckpointFile, m *MemoryStore) error {
 		b := &buffer{
 			frequency: metric.Frequency,
 			start:     metric.Start,
-			data:      metric.Data[0:n:n], // Space is wasted here :(
+			data:      metric.Data[0:n:n],
 			prev:      nil,
 			next:      nil,
 			archived:  true,
 		}
-		b.close()
 
 		minfo, ok := m.Metrics[name]
 		if !ok {
 			continue
-			// return errors.New("Unkown metric: " + name)
 		}
 
 		prev := l.metrics[minfo.offset]

internal/memorystore/config.go

@@ -7,6 +7,16 @@ package memorystore
 
 import (
 	"fmt"
+	"time"
+)
+
+const (
+	DefaultMaxWorkers             = 10
+	DefaultBufferCapacity         = 512
+	DefaultGCTriggerInterval      = 100
+	DefaultAvroWorkers            = 4
+	DefaultCheckpointBufferMin    = 3
+	DefaultAvroCheckpointInterval = time.Minute
 )
 
 var InternalCCMSFlag bool = false
@@ -14,7 +24,7 @@ var InternalCCMSFlag bool = false
 type MetricStoreConfig struct {
 	// Number of concurrent workers for checkpoint and archive operations.
 	// If not set or 0, defaults to min(runtime.NumCPU()/2+1, 10)
-	NumWorkers int `json:"num-workers"`
+	NumWorkers  int `json:"num-workers"`
 	Checkpoints struct {
 		FileFormat string `json:"file-format"`
 		Interval   string `json:"interval"`
@@ -31,20 +41,6 @@ type MetricStoreConfig struct {
 		RootDir       string `json:"directory"`
 		DeleteInstead bool   `json:"delete-instead"`
 	} `json:"archive"`
-	Nats []*NatsConfig `json:"nats"`
-}
-
-type NatsConfig struct {
-	// Address of the nats server
-	Address string `json:"address"`
-
-	// Username/Password, optional
-	Username string `json:"username"`
-	Password string `json:"password"`
-
-	// Creds file path
-	Credsfilepath string `json:"creds-file-path"`
-
 	Subscriptions []struct {
 		// Channel name
 		SubscribeTo string `json:"subscribe-to"`

internal/memorystore/level.go

@@ -46,7 +46,7 @@ func (l *Level) findLevelOrCreate(selector []string, nMetrics int) *Level {
 		}
 	}
 
-	// The level does not exist, take write lock for unqiue access:
+	// The level does not exist, take write lock for unique access:
 	l.lock.Lock()
 	// While this thread waited for the write lock, another thread
 	// could have created the child node.

internal/memorystore/lineprotocol.go

@@ -11,113 +11,36 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ClusterCockpit/cc-backend/pkg/nats"
 	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
 	"github.com/ClusterCockpit/cc-lib/schema"
 	"github.com/influxdata/line-protocol/v2/lineprotocol"
-	"github.com/nats-io/nats.go"
 )
 
-// Each connection is handled in it's own goroutine. This is a blocking function.
+func ReceiveNats(ms *MemoryStore,
-// func ReceiveRaw(ctx context.Context,
-// 	listener net.Listener,
-// 	handleLine func(*lineprotocol.Decoder, string) error,
-// ) error {
-// 	var wg sync.WaitGroup
-
-// 	wg.Add(1)
-// 	go func() {
-// 		defer wg.Done()
-// 		<-ctx.Done()
-// 		if err := listener.Close(); err != nil {
-// 			log.Printf("listener.Close(): %s", err.Error())
-// 		}
-// 	}()
-
-// 	for {
-// 		conn, err := listener.Accept()
-// 		if err != nil {
-// 			if errors.Is(err, net.ErrClosed) {
-// 				break
-// 			}
-
-// 			log.Printf("listener.Accept(): %s", err.Error())
-// 		}
-
-// 		wg.Add(2)
-// 		go func() {
-// 			defer wg.Done()
-// 			defer conn.Close()
-
-// 			dec := lineprotocol.NewDecoder(conn)
-// 			connctx, cancel := context.WithCancel(context.Background())
-// 			defer cancel()
-// 			go func() {
-// 				defer wg.Done()
-// 				select {
-// 				case <-connctx.Done():
-// 					conn.Close()
-// 				case <-ctx.Done():
-// 					conn.Close()
-// 				}
-// 			}()
-
-// 			if err := handleLine(dec, "default"); err != nil {
-// 				if errors.Is(err, net.ErrClosed) {
-// 					return
-// 				}
-
-// 				log.Printf("%s: %s", conn.RemoteAddr().String(), err.Error())
-// 				errmsg := make([]byte, 128)
-// 				errmsg = append(errmsg, `error: `...)
-// 				errmsg = append(errmsg, err.Error()...)
-// 				errmsg = append(errmsg, '\n')
-// 				conn.Write(errmsg)
-// 			}
-// 		}()
-// 	}
-
-// 	wg.Wait()
-// 	return nil
-// }
-
-// ReceiveNats connects to a nats server and subscribes to "updates". This is a
-// blocking function. handleLine will be called for each line recieved via
-// nats. Send `true` through the done channel for gracefull termination.
-func ReceiveNats(conf *(NatsConfig),
-	ms *MemoryStore,
 	workers int,
 	ctx context.Context,
 ) error {
-	var opts []nats.Option
+	nc := nats.GetClient()
-	if conf.Username != "" && conf.Password != "" {
-		opts = append(opts, nats.UserInfo(conf.Username, conf.Password))
-	}
 
-	if conf.Credsfilepath != "" {
+	if nc == nil {
-		opts = append(opts, nats.UserCredentials(conf.Credsfilepath))
+		cclog.Warn("NATS client not initialized")
+		return nil
 	}
 
-	nc, err := nats.Connect(conf.Address, opts...)
-	if err != nil {
-		return err
-	}
-	defer nc.Close()
-
 	var wg sync.WaitGroup
-	var subs []*nats.Subscription
 
-	msgs := make(chan *nats.Msg, workers*2)
+	msgs := make(chan []byte, workers*2)
 
-	for _, sc := range conf.Subscriptions {
+	for _, sc := range Keys.Subscriptions {
 		clusterTag := sc.ClusterTag
-		var sub *nats.Subscription
 		if workers > 1 {
 			wg.Add(workers)
 
 			for range workers {
 				go func() {
 					for m := range msgs {
-						dec := lineprotocol.NewDecoderWithBytes(m.Data)
+						dec := lineprotocol.NewDecoderWithBytes(m)
 						if err := DecodeLine(dec, ms, clusterTag); err != nil {
 							cclog.Errorf("error: %s", err.Error())
 						}
@@ -127,37 +50,23 @@ func ReceiveNats(conf *(NatsConfig),
 				}()
 			}
 
-			sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) {
+			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
-				msgs <- m
+				msgs <- data
 			})
 		} else {
-			sub, err = nc.Subscribe(sc.SubscribeTo, func(m *nats.Msg) {
+			nc.Subscribe(sc.SubscribeTo, func(subject string, data []byte) {
-				dec := lineprotocol.NewDecoderWithBytes(m.Data)
+				dec := lineprotocol.NewDecoderWithBytes(data)
 				if err := DecodeLine(dec, ms, clusterTag); err != nil {
 					cclog.Errorf("error: %s", err.Error())
 				}
 			})
 		}
-
+		cclog.Infof("NATS subscription to '%s' established", sc.SubscribeTo)
-		if err != nil {
-			return err
-		}
-		cclog.Infof("NATS subscription to '%s' on '%s' established", sc.SubscribeTo, conf.Address)
-		subs = append(subs, sub)
 	}
 
-	<-ctx.Done()
-	for _, sub := range subs {
-		err = sub.Unsubscribe()
-		if err != nil {
-			cclog.Errorf("NATS unsubscribe failed: %s", err.Error())
-		}
-	}
 	close(msgs)
 	wg.Wait()
 
-	nc.Close()
-	cclog.Print("NATS connection closed")
 	return nil
 }
 
@@ -266,8 +175,6 @@ func DecodeLine(dec *lineprotocol.Decoder,
 			case "stype-id":
 				subTypeBuf = append(subTypeBuf, val...)
 			default:
-				// Ignore unkown tags (cc-metric-collector might send us a unit for example that we do not need)
-				// return fmt.Errorf("unkown tag: '%s' (value: '%s')", string(key), string(val))
 			}
 		}
 

internal/memorystore/memorystore.go

@@ -44,8 +44,6 @@ var (
 	shutdownFunc context.CancelFunc
 )
 
-
-
 type Metric struct {
 	Name         string
 	Value        schema.Float
@@ -71,8 +69,7 @@ func Init(rawConfig json.RawMessage, wg *sync.WaitGroup) {
 
 	// Set NumWorkers from config or use default
 	if Keys.NumWorkers <= 0 {
-		maxWorkers := 10
+		Keys.NumWorkers = min(runtime.NumCPU()/2+1, DefaultMaxWorkers)
-		Keys.NumWorkers = min(runtime.NumCPU()/2+1, maxWorkers)
 	}
 	cclog.Debugf("[METRICSTORE]> Using %d workers for checkpoint/archive operations\n", Keys.NumWorkers)
 
@@ -144,20 +141,9 @@ func Init(rawConfig json.RawMessage, wg *sync.WaitGroup) {
 	// Store the shutdown function for later use by Shutdown()
 	shutdownFunc = shutdown
 
-	if Keys.Nats != nil {
+	err = ReceiveNats(ms, 1, ctx)
-		for _, natsConf := range Keys.Nats {
+	if err != nil {
-			// TODO: When multiple nats configs share a URL, do a single connect.
+		cclog.Fatal(err)
-			wg.Add(1)
-			nc := natsConf
-			go func() {
-				// err := ReceiveNats(conf.Nats, decodeLine, runtime.NumCPU()-1, ctx)
-				err := ReceiveNats(nc, ms, 1, ctx)
-				if err != nil {
-					cclog.Fatal(err)
-				}
-				wg.Done()
-			}()
-		}
 	}
 }
 
@@ -244,18 +230,18 @@ func Retention(wg *sync.WaitGroup, ctx context.Context) {
 			return
 		}
 
-		ticks := func() <-chan time.Time {
+		tickInterval := d / 2
-			d := d / 2
+		if tickInterval <= 0 {
-			if d <= 0 {
+			return
-				return nil
+		}
-			}
+		ticker := time.NewTicker(tickInterval)
-			return time.NewTicker(d).C
+		defer ticker.Stop()
-		}()
+
 		for {
 			select {
 			case <-ctx.Done():
 				return
-			case <-ticks:
+			case <-ticker.C:
 				t := time.Now().Add(-d)
 				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
 				freed, err := ms.Free(nil, t.Unix())
@@ -332,7 +318,7 @@ func (m *MemoryStore) Read(selector util.Selector, metric string, from, to, reso
 
 	minfo, ok := m.Metrics[metric]
 	if !ok {
-		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unkown metric: " + metric)
+		return nil, 0, 0, 0, errors.New("[METRICSTORE]> unknown metric: " + metric)
 	}
 
 	n, data := 0, make([]schema.Float, (to-from)/minfo.Frequency+1)

internal/memorystore/stats.go

@@ -77,7 +77,7 @@ func (m *MemoryStore) Stats(selector util.Selector, metric string, from, to int6
 
 	minfo, ok := m.Metrics[metric]
 	if !ok {
-		return nil, 0, 0, errors.New("unkown metric: " + metric)
+		return nil, 0, 0, errors.New("unknown metric: " + metric)
 	}
 
 	n, samples := 0, 0

internal/repository/job.go

@@ -376,7 +376,7 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64, omitTagged bool) (int,
 	return cnt, err
 }
 
-func (r *JobRepository) DeleteJobById(id int64) error {
+func (r *JobRepository) DeleteJobByID(id int64) error {
 	// Invalidate cache entries before deletion
 	r.cache.Del(fmt.Sprintf("metadata:%d", id))
 	r.cache.Del(fmt.Sprintf("energyFootprint:%d", id))
@@ -577,10 +577,10 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
 	return nil
 }
 
-func (r *JobRepository) FindJobIdsByTag(tagId int64) ([]int64, error) {
+func (r *JobRepository) FindJobIdsByTag(tagID int64) ([]int64, error) {
 	query := sq.Select("job.id").From("job").
 		Join("jobtag ON jobtag.job_id = job.id").
-		Where(sq.Eq{"jobtag.tag_id": tagId}).Distinct()
+		Where(sq.Eq{"jobtag.tag_id": tagID}).Distinct()
 	rows, err := query.RunWith(r.stmtCache).Query()
 	if err != nil {
 		cclog.Error("Error while running query")
@@ -589,15 +589,15 @@ func (r *JobRepository) FindJobIdsByTag(tagId int64) ([]int64, error) {
 	jobIds := make([]int64, 0, 100)
 
 	for rows.Next() {
-		var jobId int64
+		var jobID int64
 
-		if err := rows.Scan(&jobId); err != nil {
+		if err := rows.Scan(&jobID); err != nil {
 			rows.Close()
 			cclog.Warn("Error while scanning rows")
 			return nil, err
 		}
 
-		jobIds = append(jobIds, jobId)
+		jobIds = append(jobIds, jobID)
 	}
 
 	return jobIds, nil
@@ -731,10 +731,11 @@ func (r *JobRepository) UpdateEnergy(
 		metricEnergy := 0.0
 		if i, err := archive.MetricIndex(sc.MetricConfig, fp); err == nil {
 			// Note: For DB data, calculate and save as kWh
-			if sc.MetricConfig[i].Energy == "energy" { // this metric has energy as unit (Joules or Wh)
+			switch sc.MetricConfig[i].Energy {
+			case "energy": // this metric has energy as unit (Joules or Wh)
 				cclog.Warnf("Update EnergyFootprint for Job %d and Metric %s on cluster %s: Set to 'energy' in cluster.json: Not implemented, will return 0.0", jobMeta.JobID, jobMeta.Cluster, fp)
 				// FIXME: Needs sum as stats type
-			} else if sc.MetricConfig[i].Energy == "power" { // this metric has power as unit (Watt)
+			case "power": // this metric has power as unit (Watt)
 				// Energy: Power (in Watts) * Time (in Seconds)
 				// Unit: (W * (s / 3600)) / 1000 = kWh
 				// Round 2 Digits: round(Energy * 100) / 100

internal/repository/jobCreate.go

@@ -2,6 +2,7 @@
 // All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package repository
 
 import (
@@ -109,27 +110,27 @@ func (r *JobRepository) Start(job *schema.Job) (id int64, err error) {
 
 // Stop updates the job with the database id jobId using the provided arguments.
 func (r *JobRepository) Stop(
-	jobId int64,
+	jobID int64,
 	duration int32,
 	state schema.JobState,
 	monitoringStatus int32,
 ) (err error) {
 	// Invalidate cache entries as job state is changing
-	r.cache.Del(fmt.Sprintf("metadata:%d", jobId))
+	r.cache.Del(fmt.Sprintf("metadata:%d", jobID))
-	r.cache.Del(fmt.Sprintf("energyFootprint:%d", jobId))
+	r.cache.Del(fmt.Sprintf("energyFootprint:%d", jobID))
 
 	stmt := sq.Update("job").
 		Set("job_state", state).
 		Set("duration", duration).
 		Set("monitoring_status", monitoringStatus).
-		Where("job.id = ?", jobId)
+		Where("job.id = ?", jobID)
 
 	_, err = stmt.RunWith(r.stmtCache).Exec()
 	return err
 }
 
 func (r *JobRepository) StopCached(
-	jobId int64,
+	jobID int64,
 	duration int32,
 	state schema.JobState,
 	monitoringStatus int32,
@@ -140,7 +141,7 @@ func (r *JobRepository) StopCached(
 		Set("job_state", state).
 		Set("duration", duration).
 		Set("monitoring_status", monitoringStatus).
-		Where("job_cache.id = ?", jobId)
+		Where("job_cache.id = ?", jobID)
 
 	_, err = stmt.RunWith(r.stmtCache).Exec()
 	return err

internal/repository/jobFind.go

@@ -2,6 +2,7 @@
 // All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package repository
 
 import (
@@ -22,13 +23,13 @@ import (
 // It returns a pointer to a schema.Job data structure and an error variable.
 // To check if no job was found test err == sql.ErrNoRows
 func (r *JobRepository) Find(
-	jobId *int64,
+	jobID *int64,
 	cluster *string,
 	startTime *int64,
 ) (*schema.Job, error) {
 	start := time.Now()
 	q := sq.Select(jobColumns...).From("job").
-		Where("job.job_id = ?", *jobId)
+		Where("job.job_id = ?", *jobID)
 
 	if cluster != nil {
 		q = q.Where("job.cluster = ?", *cluster)
@@ -44,12 +45,12 @@ func (r *JobRepository) Find(
 }
 
 func (r *JobRepository) FindCached(
-	jobId *int64,
+	jobID *int64,
 	cluster *string,
 	startTime *int64,
 ) (*schema.Job, error) {
 	q := sq.Select(jobCacheColumns...).From("job_cache").
-		Where("job_cache.job_id = ?", *jobId)
+		Where("job_cache.job_id = ?", *jobID)
 
 	if cluster != nil {
 		q = q.Where("job_cache.cluster = ?", *cluster)
@@ -63,19 +64,19 @@ func (r *JobRepository) FindCached(
 	return scanJob(q.RunWith(r.stmtCache).QueryRow())
 }
 
-// Find executes a SQL query to find a specific batch job.
+// FindAll executes a SQL query to find all batch jobs matching the given criteria.
-// The job is queried using the batch job id, the cluster name,
+// Jobs are queried using the batch job id, and optionally filtered by cluster name
-// and the start time of the job in UNIX epoch time seconds.
+// and start time (UNIX epoch time seconds).
-// It returns a pointer to a schema.Job data structure and an error variable.
+// It returns a slice of pointers to schema.Job data structures and an error variable.
-// To check if no job was found test err == sql.ErrNoRows
+// An empty slice is returned if no matching jobs are found.
 func (r *JobRepository) FindAll(
-	jobId *int64,
+	jobID *int64,
 	cluster *string,
 	startTime *int64,
 ) ([]*schema.Job, error) {
 	start := time.Now()
 	q := sq.Select(jobColumns...).From("job").
-		Where("job.job_id = ?", *jobId)
+		Where("job.job_id = ?", *jobID)
 
 	if cluster != nil {
 		q = q.Where("job.cluster = ?", *cluster)
@@ -139,13 +140,13 @@ func (r *JobRepository) GetJobList(limit int, offset int) ([]int64, error) {
 	return jl, nil
 }
 
-// FindById executes a SQL query to find a specific batch job.
+// FindByID executes a SQL query to find a specific batch job.
 // The job is queried using the database id.
 // It returns a pointer to a schema.Job data structure and an error variable.
 // To check if no job was found test err == sql.ErrNoRows
-func (r *JobRepository) FindById(ctx context.Context, jobId int64) (*schema.Job, error) {
+func (r *JobRepository) FindByID(ctx context.Context, jobID int64) (*schema.Job, error) {
 	q := sq.Select(jobColumns...).
-		From("job").Where("job.id = ?", jobId)
+		From("job").Where("job.id = ?", jobID)
 
 	q, qerr := SecurityCheck(ctx, q)
 	if qerr != nil {
@@ -155,14 +156,14 @@ func (r *JobRepository) FindById(ctx context.Context, jobId int64) (*schema.Job,
 	return scanJob(q.RunWith(r.stmtCache).QueryRow())
 }
 
-// FindByIdWithUser executes a SQL query to find a specific batch job.
+// FindByIDWithUser executes a SQL query to find a specific batch job.
 // The job is queried using the database id. The user is passed directly,
 // instead as part of the context.
 // It returns a pointer to a schema.Job data structure and an error variable.
 // To check if no job was found test err == sql.ErrNoRows
-func (r *JobRepository) FindByIdWithUser(user *schema.User, jobId int64) (*schema.Job, error) {
+func (r *JobRepository) FindByIDWithUser(user *schema.User, jobID int64) (*schema.Job, error) {
 	q := sq.Select(jobColumns...).
-		From("job").Where("job.id = ?", jobId)
+		From("job").Where("job.id = ?", jobID)
 
 	q, qerr := SecurityCheckWithUser(user, q)
 	if qerr != nil {
@@ -172,24 +173,24 @@ func (r *JobRepository) FindByIdWithUser(user *schema.User, jobId int64) (*schem
 	return scanJob(q.RunWith(r.stmtCache).QueryRow())
 }
 
-// FindByIdDirect executes a SQL query to find a specific batch job.
+// FindByIDDirect executes a SQL query to find a specific batch job.
 // The job is queried using the database id.
 // It returns a pointer to a schema.Job data structure and an error variable.
 // To check if no job was found test err == sql.ErrNoRows
-func (r *JobRepository) FindByIdDirect(jobId int64) (*schema.Job, error) {
+func (r *JobRepository) FindByIDDirect(jobID int64) (*schema.Job, error) {
 	q := sq.Select(jobColumns...).
-		From("job").Where("job.id = ?", jobId)
+		From("job").Where("job.id = ?", jobID)
 	return scanJob(q.RunWith(r.stmtCache).QueryRow())
 }
 
-// FindByJobId executes a SQL query to find a specific batch job.
+// FindByJobID executes a SQL query to find a specific batch job.
 // The job is queried using the slurm id and the clustername.
 // It returns a pointer to a schema.Job data structure and an error variable.
 // To check if no job was found test err == sql.ErrNoRows
-func (r *JobRepository) FindByJobId(ctx context.Context, jobId int64, startTime int64, cluster string) (*schema.Job, error) {
+func (r *JobRepository) FindByJobID(ctx context.Context, jobID int64, startTime int64, cluster string) (*schema.Job, error) {
 	q := sq.Select(jobColumns...).
 		From("job").
-		Where("job.job_id = ?", jobId).
+		Where("job.job_id = ?", jobID).
 		Where("job.cluster = ?", cluster).
 		Where("job.start_time = ?", startTime)
 
@@ -205,10 +206,10 @@ func (r *JobRepository) FindByJobId(ctx context.Context, jobId int64, startTime
 // The job is queried using the slurm id,a username and the cluster.
 // It returns a bool.
 // If job was found, user is owner: test err != sql.ErrNoRows
-func (r *JobRepository) IsJobOwner(jobId int64, startTime int64, user string, cluster string) bool {
+func (r *JobRepository) IsJobOwner(jobID int64, startTime int64, user string, cluster string) bool {
 	q := sq.Select("id").
 		From("job").
-		Where("job.job_id = ?", jobId).
+		Where("job.job_id = ?", jobID).
 		Where("job.hpc_user = ?", user).
 		Where("job.cluster = ?", cluster).
 		Where("job.start_time = ?", startTime)
@@ -269,19 +270,19 @@ func (r *JobRepository) FindConcurrentJobs(
 	queryString := fmt.Sprintf("cluster=%s", job.Cluster)
 
 	for rows.Next() {
-		var id, jobId, startTime sql.NullInt64
+		var id, jobID, startTime sql.NullInt64
 
-		if err = rows.Scan(&id, &jobId, &startTime); err != nil {
+		if err = rows.Scan(&id, &jobID, &startTime); err != nil {
 			cclog.Warn("Error while scanning rows")
 			return nil, err
 		}
 
 		if id.Valid {
-			queryString += fmt.Sprintf("&jobId=%d", int(jobId.Int64))
+			queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64))
 			items = append(items,
 				&model.JobLink{
 					ID:    fmt.Sprint(id.Int64),
-					JobID: int(jobId.Int64),
+					JobID: int(jobID.Int64),
 				})
 		}
 	}
@@ -294,19 +295,19 @@ func (r *JobRepository) FindConcurrentJobs(
 	defer rows.Close()
 
 	for rows.Next() {
-		var id, jobId, startTime sql.NullInt64
+		var id, jobID, startTime sql.NullInt64
 
-		if err := rows.Scan(&id, &jobId, &startTime); err != nil {
+		if err := rows.Scan(&id, &jobID, &startTime); err != nil {
 			cclog.Warn("Error while scanning rows")
 			return nil, err
 		}
 
 		if id.Valid {
-			queryString += fmt.Sprintf("&jobId=%d", int(jobId.Int64))
+			queryString += fmt.Sprintf("&jobId=%d", int(jobID.Int64))
 			items = append(items,
 				&model.JobLink{
 					ID:    fmt.Sprint(id.Int64),
-					JobID: int(jobId.Int64),
+					JobID: int(jobID.Int64),
 				})
 		}
 	}

internal/repository/job_test.go

@@ -33,7 +33,7 @@ func TestFind(t *testing.T) {
 func TestFindById(t *testing.T) {
 	r := setup(t)
 
-	job, err := r.FindById(getContext(t), 338)
+	job, err := r.FindByID(getContext(t), 338)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/repository/migration.go

@@ -118,7 +118,7 @@ func MigrateDB(backend string, db string) error {
 	v, dirty, err := m.Version()
 	if err != nil {
 		if err == migrate.ErrNilVersion {
-			cclog.Warn("Legacy database without version or missing database file!")
+			cclog.Info("Legacy database without version or missing database file!")
 		} else {
 			return err
 		}

internal/repository/node.go

@@ -10,6 +10,8 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"slices"
+	"strings"
 	"sync"
 	"time"
 
@@ -106,7 +108,7 @@ func (r *NodeRepository) GetNode(hostname string, cluster string, withMeta bool)
 	return node, nil
 }
 
-func (r *NodeRepository) GetNodeById(id int64, withMeta bool) (*schema.Node, error) {
+func (r *NodeRepository) GetNodeByID(id int64, withMeta bool) (*schema.Node, error) {
 	node := &schema.Node{}
 	var timestamp int
 	if err := sq.Select("node.hostname", "node.cluster", "node.subcluster", "node_state.node_state",
@@ -240,7 +242,6 @@ func (r *NodeRepository) QueryNodes(
 	page *model.PageRequest,
 	order *model.OrderByInput, // Currently unused!
 ) ([]*schema.Node, error) {
-
 	query, qerr := AccessCheck(ctx,
 		sq.Select("hostname", "cluster", "subcluster", "node_state", "health_state", "MAX(time_stamp) as time").
 			From("node").
@@ -309,7 +310,6 @@ func (r *NodeRepository) CountNodes(
 	ctx context.Context,
 	filters []*model.NodeFilter,
 ) (int, error) {
-
 	query, qerr := AccessCheck(ctx,
 		sq.Select("time_stamp", "count(*) as countRes").
 			From("node").
@@ -553,6 +553,140 @@ func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model.
 	return timedStates, nil
 }
 
+func (r *NodeRepository) GetNodesForList(
+	ctx context.Context,
+	cluster string,
+	subCluster string,
+	stateFilter string,
+	nodeFilter string,
+	page *model.PageRequest,
+) ([]string, map[string]string, int, bool, error) {
+
+	// Init Return Vars
+	nodes := make([]string, 0)
+	stateMap := make(map[string]string)
+	countNodes := 0
+	hasNextPage := false
+
+	// Build Filters
+	queryFilters := make([]*model.NodeFilter, 0)
+	if cluster != "" {
+		queryFilters = append(queryFilters, &model.NodeFilter{Cluster: &model.StringInput{Eq: &cluster}})
+	}
+	if subCluster != "" {
+		queryFilters = append(queryFilters, &model.NodeFilter{Subcluster: &model.StringInput{Eq: &subCluster}})
+	}
+	if nodeFilter != "" && stateFilter != "notindb" {
+		queryFilters = append(queryFilters, &model.NodeFilter{Hostname: &model.StringInput{Contains: &nodeFilter}})
+	}
+	if stateFilter != "all" && stateFilter != "notindb" {
+		var queryState schema.SchedulerState = schema.SchedulerState(stateFilter)
+		queryFilters = append(queryFilters, &model.NodeFilter{SchedulerState: &queryState})
+	}
+	// if healthFilter != "all" {
+	// 	filters = append(filters, &model.NodeFilter{HealthState: &healthFilter})
+	// }
+
+	// Special Case: Disable Paging for missing nodes filter, save IPP for later
+	var backupItems int
+	if stateFilter == "notindb" {
+		backupItems = page.ItemsPerPage
+		page.ItemsPerPage = -1
+	}
+
+	// Query Nodes From DB
+	rawNodes, serr := r.QueryNodes(ctx, queryFilters, page, nil) // Order not Used
+	if serr != nil {
+		cclog.Warn("error while loading node database data (Resolver.NodeMetricsList)")
+		return nil, nil, 0, false, serr
+	}
+
+	// Intermediate Node Result Info
+	for _, node := range rawNodes {
+		if node == nil {
+			continue
+		}
+		nodes = append(nodes, node.Hostname)
+		stateMap[node.Hostname] = string(node.NodeState)
+	}
+
+	// Special Case: Find Nodes not in DB node table but in metricStore only
+	if stateFilter == "notindb" {
+		// Reapply Original Paging
+		page.ItemsPerPage = backupItems
+		// Get Nodes From Topology
+		var topoNodes []string
+		if subCluster != "" {
+			scNodes := archive.NodeLists[cluster][subCluster]
+			topoNodes = scNodes.PrintList()
+		} else {
+			subClusterNodeLists := archive.NodeLists[cluster]
+			for _, nodeList := range subClusterNodeLists {
+				topoNodes = append(topoNodes, nodeList.PrintList()...)
+			}
+		}
+		// Compare to all nodes from cluster/subcluster in DB
+		var missingNodes []string
+		for _, scanNode := range topoNodes {
+			if !slices.Contains(nodes, scanNode) {
+				missingNodes = append(missingNodes, scanNode)
+			}
+		}
+		// Filter nodes by name
+		if nodeFilter != "" {
+			filteredNodesByName := []string{}
+			for _, missingNode := range missingNodes {
+				if strings.Contains(missingNode, nodeFilter) {
+					filteredNodesByName = append(filteredNodesByName, missingNode)
+				}
+			}
+			missingNodes = filteredNodesByName
+		}
+		// Sort Missing Nodes Alphanumerically
+		slices.Sort(missingNodes)
+		// Total Missing
+		countNodes = len(missingNodes)
+		// Apply paging
+		if countNodes > page.ItemsPerPage {
+			start := (page.Page - 1) * page.ItemsPerPage
+			end := start + page.ItemsPerPage
+			if end > countNodes {
+				end = countNodes
+				hasNextPage = false
+			} else {
+				hasNextPage = true
+			}
+			nodes = missingNodes[start:end]
+		} else {
+			nodes = missingNodes
+		}
+
+	} else {
+		// DB Nodes: Count and Find Next Page
+		var cerr error
+		countNodes, cerr = r.CountNodes(ctx, queryFilters)
+		if cerr != nil {
+			cclog.Warn("error while counting node database data (Resolver.NodeMetricsList)")
+			return nil, nil, 0, false, cerr
+		}
+
+		// Example Page 4 @ 10 IpP : Does item 41 exist?
+		// Minimal Page 41 @ 1 IpP : If len(result) is 1, Page 5 exists.
+		nextPage := &model.PageRequest{
+			ItemsPerPage: 1,
+			Page:         ((page.Page * page.ItemsPerPage) + 1),
+		}
+		nextNodes, err := r.QueryNodes(ctx, queryFilters, nextPage, nil) // Order not Used
+		if err != nil {
+			cclog.Warn("Error while querying next nodes")
+			return nil, nil, 0, false, err
+		}
+		hasNextPage = len(nextNodes) == 1
+	}
+
+	return nodes, stateMap, countNodes, hasNextPage, nil
+}
+
 func AccessCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) {
 	user := GetUserFromContext(ctx)
 	return AccessCheckWithUser(user, query)

internal/repository/repository_test.go

@@ -55,7 +55,7 @@ func BenchmarkDB_FindJobById(b *testing.B) {
 
 		b.RunParallel(func(pb *testing.PB) {
 			for pb.Next() {
-				_, err := db.FindById(getContext(b), jobId)
+				_, err := db.FindByID(getContext(b), jobId)
 				noErr(b, err)
 			}
 		})

internal/repository/tags.go

@@ -2,6 +2,7 @@
 // All rights reserved. This file is part of cc-backend.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
+
 package repository
 
 import (
@@ -18,7 +19,7 @@ import (
 // AddTag adds the tag with id `tagId` to the job with the database id `jobId`.
 // Requires user authentication for security checks.
 func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*schema.Tag, error) {
-	j, err := r.FindByIdWithUser(user, job)
+	j, err := r.FindByIDWithUser(user, job)
 	if err != nil {
 		cclog.Warnf("Error finding job %d for user %s: %v", job, user.Username, err)
 		return nil, err
@@ -32,7 +33,7 @@ func (r *JobRepository) AddTag(user *schema.User, job int64, tag int64) ([]*sche
 // AddTagDirect adds a tag without user security checks.
 // Use only for internal/admin operations.
 func (r *JobRepository) AddTagDirect(job int64, tag int64) ([]*schema.Tag, error) {
-	j, err := r.FindByIdDirect(job)
+	j, err := r.FindByIDDirect(job)
 	if err != nil {
 		cclog.Warnf("Error finding job %d: %v", job, err)
 		return nil, err
@@ -43,10 +44,10 @@ func (r *JobRepository) AddTagDirect(job int64, tag int64) ([]*schema.Tag, error
 	})
 }
 
-// Removes a tag from a job by tag id.
+// RemoveTag removes the tag with the database id `tag` from the job with the database id `job`.
-// Used by GraphQL API
+// Requires user authentication for security checks. Used by GraphQL API.
 func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.Tag, error) {
-	j, err := r.FindByIdWithUser(user, job)
+	j, err := r.FindByIDWithUser(user, job)
 	if err != nil {
 		cclog.Warn("Error while finding job by id")
 		return nil, err
@@ -75,8 +76,8 @@ func (r *JobRepository) RemoveTag(user *schema.User, job, tag int64) ([]*schema.
 	return tags, archive.UpdateTags(j, archiveTags)
 }
 
-// Removes a tag from a job by tag info
+// RemoveJobTagByRequest removes a tag from the job with the database id `job` by tag type, name, and scope.
-// Used by REST API
+// Requires user authentication for security checks. Used by REST API.
 func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagType string, tagName string, tagScope string) ([]*schema.Tag, error) {
 	// Get Tag ID to delete
 	tagID, exists := r.TagId(tagType, tagName, tagScope)
@@ -86,7 +87,7 @@ func (r *JobRepository) RemoveJobTagByRequest(user *schema.User, job int64, tagT
 	}
 
 	// Get Job
-	j, err := r.FindByIdWithUser(user, job)
+	j, err := r.FindByIDWithUser(user, job)
 	if err != nil {
 		cclog.Warn("Error while finding job by id")
 		return nil, err
@@ -124,7 +125,7 @@ func (r *JobRepository) removeTagFromArchiveJobs(jobIds []int64) {
 			continue
 		}
 
-		job, err := r.FindByIdDirect(j)
+		job, err := r.FindByIDDirect(j)
 		if err != nil {
 			cclog.Warnf("Error while getting job %d", j)
 			continue

internal/routerConfig/routes.go

@@ -47,7 +47,9 @@ var routes []Route = []Route{
 	{"/monitoring/systems/list/{cluster}/{subcluster}", "monitoring/systems.tmpl", "Cluster <ID> <SID> Node List - ClusterCockpit", false, setupClusterListRoute},
 	{"/monitoring/node/{cluster}/{hostname}", "monitoring/node.tmpl", "Node <ID> - ClusterCockpit", false, setupNodeRoute},
 	{"/monitoring/analysis/{cluster}", "monitoring/analysis.tmpl", "Analysis - ClusterCockpit", true, setupAnalysisRoute},
-	{"/monitoring/status/{cluster}", "monitoring/status.tmpl", "Status of <ID> - ClusterCockpit", false, setupClusterStatusRoute},
+	{"/monitoring/status/{cluster}", "monitoring/status.tmpl", "<ID> Dashboard - ClusterCockpit", false, setupClusterStatusRoute},
+	{"/monitoring/status/detail/{cluster}", "monitoring/status.tmpl", "Status of <ID> - ClusterCockpit", false, setupClusterDetailRoute},
+	{"/monitoring/dashboard/{cluster}", "monitoring/dashboard.tmpl", "<ID> Dashboard - ClusterCockpit", false, setupDashboardRoute},
 }
 
 func setupHomeRoute(i InfoType, r *http.Request) InfoType {
@@ -117,11 +119,23 @@ func setupClusterStatusRoute(i InfoType, r *http.Request) InfoType {
 	vars := mux.Vars(r)
 	i["id"] = vars["cluster"]
 	i["cluster"] = vars["cluster"]
-	from, to := r.URL.Query().Get("from"), r.URL.Query().Get("to")
+	i["displayType"] = "DASHBOARD"
-	if from != "" || to != "" {
+	return i
-		i["from"] = from
+}
-		i["to"] = to
+
-	}
+func setupClusterDetailRoute(i InfoType, r *http.Request) InfoType {
+	vars := mux.Vars(r)
+	i["id"] = vars["cluster"]
+	i["cluster"] = vars["cluster"]
+	i["displayType"] = "DETAILS"
+	return i
+}
+
+func setupDashboardRoute(i InfoType, r *http.Request) InfoType {
+	vars := mux.Vars(r)
+	i["id"] = vars["cluster"]
+	i["cluster"] = vars["cluster"]
+	i["displayType"] = "PUBLIC" // Used in Main Template
 	return i
 }
 

internal/tagger/detectApp_test.go

@@ -43,7 +43,7 @@ func TestRegister(t *testing.T) {
 func TestMatch(t *testing.T) {
 	r := setup(t)
 
-	job, err := r.FindByIdDirect(317)
+	job, err := r.FindByIDDirect(317)
 	noErr(t, err)
 
 	var tagger AppTagger

internal/tagger/tagger.go

@@ -40,7 +40,7 @@ type JobTagger struct {
 	// startTaggers are applied when a job starts (e.g., application detection)
 	startTaggers []Tagger
 	// stopTaggers are applied when a job completes (e.g., job classification)
-	stopTaggers  []Tagger
+	stopTaggers []Tagger
 }
 
 func newTagger() {
@@ -98,7 +98,7 @@ func RunTaggers() error {
 	}
 
 	for _, id := range jl {
-		job, err := r.FindByIdDirect(id)
+		job, err := r.FindByIDDirect(id)
 		if err != nil {
 			cclog.Errorf("Error while getting job %s", err)
 			return err

internal/tagger/tagger_test.go

@@ -18,7 +18,7 @@ func TestInit(t *testing.T) {
 func TestJobStartCallback(t *testing.T) {
 	Init()
 	r := setup(t)
-	job, err := r.FindByIdDirect(525)
+	job, err := r.FindByIDDirect(525)
 	noErr(t, err)
 
 	jobs := make([]*schema.Job, 0, 1)

internal/taskmanager/compressionService.go

@@ -17,7 +17,7 @@ import (
 func RegisterCompressionService(compressOlderThan int) {
 	cclog.Info("Register compression service")
 
-	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o5, 0, 0))),
+	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(5, 0, 0))),
 		gocron.NewTask(
 			func() {
 				var jobs []*schema.Job

internal/taskmanager/retentionService.go

@@ -16,7 +16,7 @@ import (
 func RegisterRetentionDeleteService(age int, includeDB bool, omitTagged bool) {
 	cclog.Info("Register retention delete service")
 
-	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o4, 0, 0))),
+	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(3, 0, 0))),
 		gocron.NewTask(
 			func() {
 				startTime := time.Now().Unix() - int64(age*24*3600)
@@ -43,7 +43,7 @@ func RegisterRetentionDeleteService(age int, includeDB bool, omitTagged bool) {
 func RegisterRetentionMoveService(age int, includeDB bool, location string, omitTagged bool) {
 	cclog.Info("Register retention move service")
 
-	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o4, 0, 0))),
+	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(4, 0, 0))),
 		gocron.NewTask(
 			func() {
 				startTime := time.Now().Unix() - int64(age*24*3600)

internal/taskmanager/stopJobsExceedTime.go

@@ -16,7 +16,7 @@ import (
 func RegisterStopJobsExceedTime() {
 	cclog.Info("Register undead jobs service")
 
-	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(0o3, 0, 0))),
+	s.NewJob(gocron.DailyJob(1, gocron.NewAtTimes(gocron.NewAtTime(3, 0, 0))),
 		gocron.NewTask(
 			func() {
 				err := jobRepo.StopJobsExceedingWalltimeBy(config.Keys.StopJobsExceedingWalltime)

pkg/archive/fsBackend.go

@@ -18,6 +18,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"text/tabwriter"
 	"time"
 
@@ -490,7 +491,46 @@ func (fsa *FsArchive) LoadClusterCfg(name string) (*schema.Cluster, error) {
 
 func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer {
 	ch := make(chan JobContainer)
+
 	go func() {
+		defer close(ch)
+
+		numWorkers := 4
+		jobPaths := make(chan string, numWorkers*2)
+		var wg sync.WaitGroup
+
+		for range numWorkers {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for jobPath := range jobPaths {
+					job, err := loadJobMeta(filepath.Join(jobPath, "meta.json"))
+					if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) {
+						cclog.Errorf("in %s: %s", jobPath, err.Error())
+						continue
+					}
+
+					if loadMetricData {
+						isCompressed := true
+						filename := filepath.Join(jobPath, "data.json.gz")
+
+						if !util.CheckFileExists(filename) {
+							filename = filepath.Join(jobPath, "data.json")
+							isCompressed = false
+						}
+
+						data, err := loadJobData(filename, isCompressed)
+						if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) {
+							cclog.Errorf("in %s: %s", jobPath, err.Error())
+						}
+						ch <- JobContainer{Meta: job, Data: &data}
+					} else {
+						ch <- JobContainer{Meta: job, Data: nil}
+					}
+				}
+			}()
+		}
+
 		clustersDir, err := os.ReadDir(fsa.path)
 		if err != nil {
 			cclog.Fatalf("Reading clusters failed @ cluster dirs: %s", err.Error())
@@ -507,7 +547,6 @@ func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer {
 
 			for _, lvl1Dir := range lvl1Dirs {
 				if !lvl1Dir.IsDir() {
-					// Could be the cluster.json file
 					continue
 				}
 
@@ -525,35 +564,17 @@ func (fsa *FsArchive) Iter(loadMetricData bool) <-chan JobContainer {
 
 					for _, startTimeDir := range startTimeDirs {
 						if startTimeDir.IsDir() {
-							job, err := loadJobMeta(filepath.Join(dirpath, startTimeDir.Name(), "meta.json"))
+							jobPaths <- filepath.Join(dirpath, startTimeDir.Name())
-							if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) {
-								cclog.Errorf("in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error())
-							}
-
-							if loadMetricData {
-								isCompressed := true
-								filename := filepath.Join(dirpath, startTimeDir.Name(), "data.json.gz")
-
-								if !util.CheckFileExists(filename) {
-									filename = filepath.Join(dirpath, startTimeDir.Name(), "data.json")
-									isCompressed = false
-								}
-
-								data, err := loadJobData(filename, isCompressed)
-								if err != nil && !errors.Is(err, &jsonschema.ValidationError{}) {
-									cclog.Errorf("in %s: %s", filepath.Join(dirpath, startTimeDir.Name()), err.Error())
-								}
-								ch <- JobContainer{Meta: job, Data: &data}
-							} else {
-								ch <- JobContainer{Meta: job, Data: nil}
-							}
 						}
 					}
 				}
 			}
 		}
-		close(ch)
+
+		close(jobPaths)
+		wg.Wait()
 	}()
+
 	return ch
 }
 
@@ -603,19 +624,52 @@ func (fsa *FsArchive) ImportJob(
 		return err
 	}
 
-	f, err = os.Create(path.Join(dir, "data.json"))
+	var dataBuf bytes.Buffer
-	if err != nil {
+	if err := EncodeJobData(&dataBuf, jobData); err != nil {
-		cclog.Error("Error while creating filepath for data.json")
+		cclog.Error("Error while encoding job metricdata")
 		return err
 	}
-	if err := EncodeJobData(f, jobData); err != nil {
+
-		cclog.Error("Error while encoding job metricdata to data.json file")
+	if dataBuf.Len() > 2000 {
-		return err
+		f, err = os.Create(path.Join(dir, "data.json.gz"))
+		if err != nil {
+			cclog.Error("Error while creating filepath for data.json.gz")
+			return err
+		}
+		gzipWriter := gzip.NewWriter(f)
+		if _, err := gzipWriter.Write(dataBuf.Bytes()); err != nil {
+			cclog.Error("Error while writing compressed job data")
+			gzipWriter.Close()
+			f.Close()
+			return err
+		}
+		if err := gzipWriter.Close(); err != nil {
+			cclog.Warn("Error while closing gzip writer")
+			f.Close()
+			return err
+		}
+		if err := f.Close(); err != nil {
+			cclog.Warn("Error while closing data.json.gz file")
+			return err
+		}
+	} else {
+		f, err = os.Create(path.Join(dir, "data.json"))
+		if err != nil {
+			cclog.Error("Error while creating filepath for data.json")
+			return err
+		}
+		if _, err := f.Write(dataBuf.Bytes()); err != nil {
+			cclog.Error("Error while writing job metricdata to data.json file")
+			f.Close()
+			return err
+		}
+		if err := f.Close(); err != nil {
+			cclog.Warn("Error while closing data.json file")
+			return err
+		}
 	}
-	if err := f.Close(); err != nil {
+
-		cclog.Warn("Error while closing data.json file")
+	return nil
-	}
-	return err
 }
 
 func (fsa *FsArchive) StoreClusterCfg(name string, config *schema.Cluster) error {

pkg/archive/s3Backend.go

@@ -17,6 +17,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"text/tabwriter"
 	"time"
 
@@ -467,7 +468,6 @@ func (s3a *S3Archive) StoreJobMeta(job *schema.Job) error {
 func (s3a *S3Archive) ImportJob(jobMeta *schema.Job, jobData *schema.JobData) error {
 	ctx := context.Background()
 
-	// Upload meta.json
 	metaKey := getS3Key(jobMeta, "meta.json")
 	var metaBuf bytes.Buffer
 	if err := EncodeJobMeta(&metaBuf, jobMeta); err != nil {
@@ -485,18 +485,37 @@ func (s3a *S3Archive) ImportJob(jobMeta *schema.Job, jobData *schema.JobData) er
 		return err
 	}
 
-	// Upload data.json
-	dataKey := getS3Key(jobMeta, "data.json")
 	var dataBuf bytes.Buffer
 	if err := EncodeJobData(&dataBuf, jobData); err != nil {
 		cclog.Error("S3Archive ImportJob() > encoding data error")
 		return err
 	}
 
+	var dataKey string
+	var dataBytes []byte
+
+	if dataBuf.Len() > 2000 {
+		dataKey = getS3Key(jobMeta, "data.json.gz")
+		var compressedBuf bytes.Buffer
+		gzipWriter := gzip.NewWriter(&compressedBuf)
+		if _, err := gzipWriter.Write(dataBuf.Bytes()); err != nil {
+			cclog.Errorf("S3Archive ImportJob() > gzip write error: %v", err)
+			return err
+		}
+		if err := gzipWriter.Close(); err != nil {
+			cclog.Errorf("S3Archive ImportJob() > gzip close error: %v", err)
+			return err
+		}
+		dataBytes = compressedBuf.Bytes()
+	} else {
+		dataKey = getS3Key(jobMeta, "data.json")
+		dataBytes = dataBuf.Bytes()
+	}
+
 	_, err = s3a.client.PutObject(ctx, &s3.PutObjectInput{
 		Bucket: aws.String(s3a.bucket),
 		Key:    aws.String(dataKey),
-		Body:   bytes.NewReader(dataBuf.Bytes()),
+		Body:   bytes.NewReader(dataBytes),
 	})
 	if err != nil {
 		cclog.Errorf("S3Archive ImportJob() > PutObject data error: %v", err)
@@ -795,29 +814,18 @@ func (s3a *S3Archive) Iter(loadMetricData bool) <-chan JobContainer {
 		ctx := context.Background()
 		defer close(ch)
 
-		for _, cluster := range s3a.clusters {
+		numWorkers := 4
-			prefix := cluster + "/"
+		metaKeys := make(chan string, numWorkers*2)
+		var wg sync.WaitGroup
 
-			paginator := s3.NewListObjectsV2Paginator(s3a.client, &s3.ListObjectsV2Input{
+		for range numWorkers {
-				Bucket: aws.String(s3a.bucket),
+			wg.Add(1)
-				Prefix: aws.String(prefix),
+			go func() {
-			})
+				defer wg.Done()
-
+				for metaKey := range metaKeys {
-			for paginator.HasMorePages() {
-				page, err := paginator.NextPage(ctx)
-				if err != nil {
-					cclog.Fatalf("S3Archive Iter() > list error: %s", err.Error())
-				}
-
-				for _, obj := range page.Contents {
-					if obj.Key == nil || !strings.HasSuffix(*obj.Key, "/meta.json") {
-						continue
-					}
-
-					// Load job metadata
 					result, err := s3a.client.GetObject(ctx, &s3.GetObjectInput{
 						Bucket: aws.String(s3a.bucket),
-						Key:    obj.Key,
+						Key:    aws.String(metaKey),
 					})
 					if err != nil {
 						cclog.Errorf("S3Archive Iter() > GetObject meta error: %v", err)
@@ -849,8 +857,34 @@ func (s3a *S3Archive) Iter(loadMetricData bool) <-chan JobContainer {
 						ch <- JobContainer{Meta: job, Data: nil}
 					}
 				}
+			}()
+		}
+
+		for _, cluster := range s3a.clusters {
+			prefix := cluster + "/"
+
+			paginator := s3.NewListObjectsV2Paginator(s3a.client, &s3.ListObjectsV2Input{
+				Bucket: aws.String(s3a.bucket),
+				Prefix: aws.String(prefix),
+			})
+
+			for paginator.HasMorePages() {
+				page, err := paginator.NextPage(ctx)
+				if err != nil {
+					cclog.Fatalf("S3Archive Iter() > list error: %s", err.Error())
+				}
+
+				for _, obj := range page.Contents {
+					if obj.Key == nil || !strings.HasSuffix(*obj.Key, "/meta.json") {
+						continue
+					}
+					metaKeys <- *obj.Key
+				}
 			}
 		}
+
+		close(metaKeys)
+		wg.Wait()
 	}()
 
 	return ch

pkg/archive/sqliteBackend.go

@@ -16,6 +16,7 @@ import (
 	"os"
 	"slices"
 	"strconv"
+	"sync"
 	"text/tabwriter"
 	"time"
 
@@ -60,6 +61,7 @@ CREATE TABLE IF NOT EXISTS jobs (
 
 CREATE INDEX IF NOT EXISTS idx_jobs_cluster ON jobs(cluster);
 CREATE INDEX IF NOT EXISTS idx_jobs_start_time ON jobs(start_time);
+CREATE INDEX IF NOT EXISTS idx_jobs_order ON jobs(cluster, start_time);
 CREATE INDEX IF NOT EXISTS idx_jobs_lookup ON jobs(cluster, job_id, start_time);
 
 CREATE TABLE IF NOT EXISTS clusters (
@@ -361,16 +363,37 @@ func (sa *SqliteArchive) ImportJob(jobMeta *schema.Job, jobData *schema.JobData)
 		return err
 	}
 
+	var dataBytes []byte
+	var compressed bool
+
+	if dataBuf.Len() > 2000 {
+		var compressedBuf bytes.Buffer
+		gzipWriter := gzip.NewWriter(&compressedBuf)
+		if _, err := gzipWriter.Write(dataBuf.Bytes()); err != nil {
+			cclog.Errorf("SqliteArchive ImportJob() > gzip write error: %v", err)
+			return err
+		}
+		if err := gzipWriter.Close(); err != nil {
+			cclog.Errorf("SqliteArchive ImportJob() > gzip close error: %v", err)
+			return err
+		}
+		dataBytes = compressedBuf.Bytes()
+		compressed = true
+	} else {
+		dataBytes = dataBuf.Bytes()
+		compressed = false
+	}
+
 	now := time.Now().Unix()
 	_, err := sa.db.Exec(`
 		INSERT INTO jobs (job_id, cluster, start_time, meta_json, data_json, data_compressed, created_at, updated_at)
-		VALUES (?, ?, ?, ?, ?, 0, ?, ?)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?)
 		ON CONFLICT(job_id, cluster, start_time) DO UPDATE SET
 			meta_json = excluded.meta_json,
 			data_json = excluded.data_json,
 			data_compressed = excluded.data_compressed,
 			updated_at = excluded.updated_at
-	`, jobMeta.JobID, jobMeta.Cluster, jobMeta.StartTime, metaBuf.Bytes(), dataBuf.Bytes(), now, now)
+	`, jobMeta.JobID, jobMeta.Cluster, jobMeta.StartTime, metaBuf.Bytes(), dataBytes, compressed, now, now)
 	if err != nil {
 		cclog.Errorf("SqliteArchive ImportJob() > insert error: %v", err)
 		return err
@@ -526,62 +549,115 @@ func (sa *SqliteArchive) CompressLast(starttime int64) int64 {
 	return last
 }
 
+type sqliteJobRow struct {
+	metaBlob   []byte
+	dataBlob   []byte
+	compressed bool
+}
+
 func (sa *SqliteArchive) Iter(loadMetricData bool) <-chan JobContainer {
 	ch := make(chan JobContainer)
 
 	go func() {
 		defer close(ch)
 
-		rows, err := sa.db.Query("SELECT job_id, cluster, start_time, meta_json, data_json, data_compressed FROM jobs ORDER BY cluster, start_time")
+		const chunkSize = 1000
-		if err != nil {
+		offset := 0
-			cclog.Fatalf("SqliteArchive Iter() > query error: %s", err.Error())
+
+		var query string
+		if loadMetricData {
+			query = "SELECT meta_json, data_json, data_compressed FROM jobs ORDER BY cluster, start_time LIMIT ? OFFSET ?"
+		} else {
+			query = "SELECT meta_json FROM jobs ORDER BY cluster, start_time LIMIT ? OFFSET ?"
 		}
-		defer rows.Close()
 
-		for rows.Next() {
+		numWorkers := 4
-			var jobID int64
+		jobRows := make(chan sqliteJobRow, numWorkers*2)
-			var cluster string
+		var wg sync.WaitGroup
-			var startTime int64
-			var metaBlob []byte
-			var dataBlob []byte
-			var compressed bool
 
-			if err := rows.Scan(&jobID, &cluster, &startTime, &metaBlob, &dataBlob, &compressed); err != nil {
+		for range numWorkers {
-				cclog.Errorf("SqliteArchive Iter() > scan error: %v", err)
+			wg.Add(1)
-				continue
+			go func() {
-			}
+				defer wg.Done()
-
+				for row := range jobRows {
-			job, err := DecodeJobMeta(bytes.NewReader(metaBlob))
+					job, err := DecodeJobMeta(bytes.NewReader(row.metaBlob))
-			if err != nil {
-				cclog.Errorf("SqliteArchive Iter() > decode meta error: %v", err)
-				continue
-			}
-
-			if loadMetricData && dataBlob != nil {
-				var reader io.Reader = bytes.NewReader(dataBlob)
-				if compressed {
-					gzipReader, err := gzip.NewReader(reader)
 					if err != nil {
-						cclog.Errorf("SqliteArchive Iter() > gzip error: %v", err)
+						cclog.Errorf("SqliteArchive Iter() > decode meta error: %v", err)
-						ch <- JobContainer{Meta: job, Data: nil}
 						continue
 					}
-					defer gzipReader.Close()
+
-					reader = gzipReader
+					if loadMetricData && row.dataBlob != nil {
+						var reader io.Reader = bytes.NewReader(row.dataBlob)
+						if row.compressed {
+							gzipReader, err := gzip.NewReader(reader)
+							if err != nil {
+								cclog.Errorf("SqliteArchive Iter() > gzip error: %v", err)
+								ch <- JobContainer{Meta: job, Data: nil}
+								continue
+							}
+							decompressed, err := io.ReadAll(gzipReader)
+							gzipReader.Close()
+							if err != nil {
+								cclog.Errorf("SqliteArchive Iter() > decompress error: %v", err)
+								ch <- JobContainer{Meta: job, Data: nil}
+								continue
+							}
+							reader = bytes.NewReader(decompressed)
+						}
+
+						key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime)
+						jobData, err := DecodeJobData(reader, key)
+						if err != nil {
+							cclog.Errorf("SqliteArchive Iter() > decode data error: %v", err)
+							ch <- JobContainer{Meta: job, Data: nil}
+						} else {
+							ch <- JobContainer{Meta: job, Data: &jobData}
+						}
+					} else {
+						ch <- JobContainer{Meta: job, Data: nil}
+					}
+				}
+			}()
+		}
+
+		for {
+			rows, err := sa.db.Query(query, chunkSize, offset)
+			if err != nil {
+				cclog.Fatalf("SqliteArchive Iter() > query error: %s", err.Error())
+			}
+
+			rowCount := 0
+			for rows.Next() {
+				var row sqliteJobRow
+
+				if loadMetricData {
+					if err := rows.Scan(&row.metaBlob, &row.dataBlob, &row.compressed); err != nil {
+						cclog.Errorf("SqliteArchive Iter() > scan error: %v", err)
+						continue
+					}
+				} else {
+					if err := rows.Scan(&row.metaBlob); err != nil {
+						cclog.Errorf("SqliteArchive Iter() > scan error: %v", err)
+						continue
+					}
+					row.dataBlob = nil
+					row.compressed = false
 				}
 
-				key := fmt.Sprintf("%s:%d:%d", job.Cluster, job.JobID, job.StartTime)
+				jobRows <- row
-				jobData, err := DecodeJobData(reader, key)
+				rowCount++
-				if err != nil {
-					cclog.Errorf("SqliteArchive Iter() > decode data error: %v", err)
-					ch <- JobContainer{Meta: job, Data: nil}
-				} else {
-					ch <- JobContainer{Meta: job, Data: &jobData}
-				}
-			} else {
-				ch <- JobContainer{Meta: job, Data: nil}
 			}
+			rows.Close()
+
+			if rowCount < chunkSize {
+				break
+			}
+
+			offset += chunkSize
 		}
+
+		close(jobRows)
+		wg.Wait()
 	}()
 
 	return ch

pkg/archive/sqliteBackend_test.go

@@ -311,3 +311,58 @@ func TestSqliteConfigParsing(t *testing.T) {
 		t.Errorf("expected dbPath '/tmp/test.db', got '%s'", cfg.DBPath)
 	}
 }
+
+func TestSqliteIterChunking(t *testing.T) {
+	tmpfile := t.TempDir() + "/test.db"
+	defer os.Remove(tmpfile)
+
+	var sa SqliteArchive
+	_, err := sa.Init(json.RawMessage(`{"dbPath":"` + tmpfile + `"}`))
+	if err != nil {
+		t.Fatalf("init failed: %v", err)
+	}
+	defer sa.db.Close()
+
+	const totalJobs = 2500
+	for i := 1; i <= totalJobs; i++ {
+		job := &schema.Job{
+			JobID:     int64(i),
+			Cluster:   "test",
+			StartTime: int64(i * 1000),
+			NumNodes:  1,
+			Resources: []*schema.Resource{{Hostname: "node001"}},
+		}
+		if err := sa.StoreJobMeta(job); err != nil {
+			t.Fatalf("store failed: %v", err)
+		}
+	}
+
+	t.Run("IterWithoutData", func(t *testing.T) {
+		count := 0
+		for container := range sa.Iter(false) {
+			if container.Meta == nil {
+				t.Error("expected non-nil meta")
+			}
+			if container.Data != nil {
+				t.Error("expected nil data when loadMetricData is false")
+			}
+			count++
+		}
+		if count != totalJobs {
+			t.Errorf("expected %d jobs, got %d", totalJobs, count)
+		}
+	})
+
+	t.Run("IterWithData", func(t *testing.T) {
+		count := 0
+		for container := range sa.Iter(true) {
+			if container.Meta == nil {
+				t.Error("expected non-nil meta")
+			}
+			count++
+		}
+		if count != totalJobs {
+			t.Errorf("expected %d jobs, got %d", totalJobs, count)
+		}
+	})
+}

pkg/nats/client.go

@@ -0,0 +1,246 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// Package nats provides a generic NATS messaging client for publish/subscribe communication.
+//
+// The package wraps the nats.go library with connection management, automatic reconnection
+// handling, and subscription tracking. It supports multiple authentication methods including
+// username/password and credential files.
+//
+// # Configuration
+//
+// Configure the client via JSON in the application config:
+//
+//	{
+//	  "nats": {
+//	    "address": "nats://localhost:4222",
+//	    "username": "user",
+//	    "password": "secret"
+//	  }
+//	}
+//
+// Or using a credentials file:
+//
+//	{
+//	  "nats": {
+//	    "address": "nats://localhost:4222",
+//	    "creds-file-path": "/path/to/creds.json"
+//	  }
+//	}
+//
+// # Usage
+//
+// The package provides a singleton client initialized once and retrieved globally:
+//
+//	nats.Init(rawConfig)
+//	nats.Connect()
+//
+//	client := nats.GetClient()
+//	client.Subscribe("events", func(subject string, data []byte) {
+//	    fmt.Printf("Received: %s\n", data)
+//	})
+//
+//	client.Publish("events", []byte("hello"))
+//
+// # Thread Safety
+//
+// All Client methods are safe for concurrent use.
+package nats
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+	"github.com/nats-io/nats.go"
+)
+
+var (
+	clientOnce     sync.Once
+	clientInstance *Client
+)
+
+// Client wraps a NATS connection with subscription management.
+type Client struct {
+	conn          *nats.Conn
+	subscriptions []*nats.Subscription
+	mu            sync.Mutex
+}
+
+// MessageHandler is a callback function for processing received messages.
+type MessageHandler func(subject string, data []byte)
+
+// Connect initializes the singleton NATS client using the global Keys config.
+func Connect() {
+	clientOnce.Do(func() {
+		if Keys.Address == "" {
+			cclog.Warn("NATS: no address configured, skipping connection")
+			return
+		}
+
+		client, err := NewClient(nil)
+		if err != nil {
+			cclog.Errorf("NATS connection failed: %v", err)
+			return
+		}
+
+		clientInstance = client
+	})
+}
+
+// GetClient returns the singleton NATS client instance.
+func GetClient() *Client {
+	if clientInstance == nil {
+		cclog.Warn("NATS client not initialized")
+	}
+	return clientInstance
+}
+
+// NewClient creates a new NATS client. If cfg is nil, uses the global Keys config.
+func NewClient(cfg *NatsConfig) (*Client, error) {
+	if cfg == nil {
+		cfg = &Keys
+	}
+
+	if cfg.Address == "" {
+		return nil, fmt.Errorf("NATS address is required")
+	}
+
+	var opts []nats.Option
+
+	if cfg.Username != "" && cfg.Password != "" {
+		opts = append(opts, nats.UserInfo(cfg.Username, cfg.Password))
+	}
+
+	if cfg.CredsFilePath != "" {
+		opts = append(opts, nats.UserCredentials(cfg.CredsFilePath))
+	}
+
+	opts = append(opts, nats.DisconnectErrHandler(func(_ *nats.Conn, err error) {
+		if err != nil {
+			cclog.Warnf("NATS disconnected: %v", err)
+		}
+	}))
+
+	opts = append(opts, nats.ReconnectHandler(func(nc *nats.Conn) {
+		cclog.Infof("NATS reconnected to %s", nc.ConnectedUrl())
+	}))
+
+	opts = append(opts, nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {
+		cclog.Errorf("NATS error: %v", err)
+	}))
+
+	nc, err := nats.Connect(cfg.Address, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("NATS connect failed: %w", err)
+	}
+
+	cclog.Infof("NATS connected to %s", cfg.Address)
+
+	return &Client{
+		conn:          nc,
+		subscriptions: make([]*nats.Subscription, 0),
+	}, nil
+}
+
+// Subscribe registers a handler for messages on the given subject.
+func (c *Client) Subscribe(subject string, handler MessageHandler) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	sub, err := c.conn.Subscribe(subject, func(msg *nats.Msg) {
+		handler(msg.Subject, msg.Data)
+	})
+	if err != nil {
+		return fmt.Errorf("NATS subscribe to '%s' failed: %w", subject, err)
+	}
+
+	c.subscriptions = append(c.subscriptions, sub)
+	cclog.Infof("NATS subscribed to '%s'", subject)
+	return nil
+}
+
+// SubscribeQueue registers a handler with queue group for load-balanced message processing.
+func (c *Client) SubscribeQueue(subject, queue string, handler MessageHandler) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	sub, err := c.conn.QueueSubscribe(subject, queue, func(msg *nats.Msg) {
+		handler(msg.Subject, msg.Data)
+	})
+	if err != nil {
+		return fmt.Errorf("NATS queue subscribe to '%s' (queue: %s) failed: %w", subject, queue, err)
+	}
+
+	c.subscriptions = append(c.subscriptions, sub)
+	cclog.Infof("NATS queue subscribed to '%s' (queue: %s)", subject, queue)
+	return nil
+}
+
+// SubscribeChan subscribes to a subject and delivers messages to the provided channel.
+func (c *Client) SubscribeChan(subject string, ch chan *nats.Msg) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	sub, err := c.conn.ChanSubscribe(subject, ch)
+	if err != nil {
+		return fmt.Errorf("NATS chan subscribe to '%s' failed: %w", subject, err)
+	}
+
+	c.subscriptions = append(c.subscriptions, sub)
+	cclog.Infof("NATS chan subscribed to '%s'", subject)
+	return nil
+}
+
+// Publish sends data to the specified subject.
+func (c *Client) Publish(subject string, data []byte) error {
+	if err := c.conn.Publish(subject, data); err != nil {
+		return fmt.Errorf("NATS publish to '%s' failed: %w", subject, err)
+	}
+	return nil
+}
+
+// Request sends a request and waits for a response with the given context timeout.
+func (c *Client) Request(subject string, data []byte, timeout context.Context) ([]byte, error) {
+	msg, err := c.conn.RequestWithContext(timeout, subject, data)
+	if err != nil {
+		return nil, fmt.Errorf("NATS request to '%s' failed: %w", subject, err)
+	}
+	return msg.Data, nil
+}
+
+// Flush flushes the connection buffer to ensure all published messages are sent.
+func (c *Client) Flush() error {
+	return c.conn.Flush()
+}
+
+// Close unsubscribes all subscriptions and closes the NATS connection.
+func (c *Client) Close() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for _, sub := range c.subscriptions {
+		if err := sub.Unsubscribe(); err != nil {
+			cclog.Warnf("NATS unsubscribe failed: %v", err)
+		}
+	}
+	c.subscriptions = nil
+
+	if c.conn != nil {
+		c.conn.Close()
+		cclog.Info("NATS connection closed")
+	}
+}
+
+// IsConnected returns true if the client has an active connection.
+func (c *Client) IsConnected() bool {
+	return c.conn != nil && c.conn.IsConnected()
+}
+
+// Connection returns the underlying NATS connection for advanced usage.
+func (c *Client) Connection() *nats.Conn {
+	return c.conn
+}

pkg/nats/config.go

@@ -0,0 +1,63 @@
+// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved. This file is part of cc-backend.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package nats
+
+import (
+	"bytes"
+	"encoding/json"
+
+	cclog "github.com/ClusterCockpit/cc-lib/ccLogger"
+)
+
+// NatsConfig holds the configuration for connecting to a NATS server.
+type NatsConfig struct {
+	Address       string `json:"address"`         // NATS server address (e.g., "nats://localhost:4222")
+	Username      string `json:"username"`        // Username for authentication (optional)
+	Password      string `json:"password"`        // Password for authentication (optional)
+	CredsFilePath string `json:"creds-file-path"` // Path to credentials file (optional)
+}
+
+// Keys holds the global NATS configuration loaded via Init.
+var Keys NatsConfig
+
+const ConfigSchema = `{
+    "type": "object",
+    "description": "Configuration for NATS messaging client.",
+    "properties": {
+        "address": {
+            "description": "Address of the NATS server (e.g., 'nats://localhost:4222').",
+            "type": "string"
+        },
+        "username": {
+            "description": "Username for NATS authentication (optional).",
+            "type": "string"
+        },
+        "password": {
+            "description": "Password for NATS authentication (optional).",
+            "type": "string"
+        },
+        "creds-file-path": {
+            "description": "Path to NATS credentials file for authentication (optional).",
+            "type": "string"
+        }
+    },
+    "required": ["address"]
+}`
+
+// Init initializes the global Keys configuration from JSON.
+func Init(rawConfig json.RawMessage) error {
+	var err error
+
+	if rawConfig != nil {
+		dec := json.NewDecoder(bytes.NewReader(rawConfig))
+		dec.DisallowUnknownFields()
+		if err = dec.Decode(&Keys); err != nil {
+			cclog.Errorf("Error while initializing nats client: %s", err.Error())
+		}
+	}
+
+	return err
+}

startDemo.sh

@@ -7,13 +7,16 @@ if [ -d './var' ]; then
   ./cc-backend -server -dev
 else
   make
-  ./cc-backend --init
+  wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-dev.tar
+  tar xf job-archive-dev.tar
+  rm ./job-archive-dev.tar
+
+  cp ./configs/env-template.txt .env
   cp ./configs/config-demo.json config.json
 
-  wget https://hpc-mover.rrze.uni-erlangen.de/HPC-Data/0x7b58aefb/eig7ahyo6fo2bais0ephuf2aitohv1ai/job-archive-demo.tar
+  ./cc-backend -migrate-db
-  tar xf job-archive-demo.tar
-  rm ./job-archive-demo.tar
-
   ./cc-backend -dev -init-db -add-user demo:admin,api:demo
+
   ./cc-backend -server -dev
+
 fi

tools.go

@@ -1,9 +0,0 @@
-//go:build tools
-// +build tools
-
-package tools
-
-import (
-	_ "github.com/99designs/gqlgen"
-	_ "github.com/swaggo/swag/cmd/swag"
-)

tools/archive-manager/import_test.go

@@ -48,7 +48,7 @@ func TestImportFileToSqlite(t *testing.T) {
 	}
 
 	// Perform import
-	imported, failed, err := importArchive(srcBackend, dstBackend)
+	imported, failed, err := importArchive(srcBackend, dstBackend, srcConfig)
 	if err != nil {
 		t.Errorf("Import failed: %s", err.Error())
 	}
@@ -111,13 +111,13 @@ func TestImportFileToFile(t *testing.T) {
 	}
 
 	// Create destination archive directory
-	if err := os.MkdirAll(dstArchive, 0755); err != nil {
+	if err := os.MkdirAll(dstArchive, 0o755); err != nil {
 		t.Fatalf("Failed to create destination directory: %s", err.Error())
 	}
 
 	// Write version file
 	versionFile := filepath.Join(dstArchive, "version.txt")
-	if err := os.WriteFile(versionFile, []byte("3"), 0644); err != nil {
+	if err := os.WriteFile(versionFile, []byte("3"), 0o644); err != nil {
 		t.Fatalf("Failed to write version file: %s", err.Error())
 	}
 
@@ -136,7 +136,7 @@ func TestImportFileToFile(t *testing.T) {
 	}
 
 	// Perform import
-	imported, failed, err := importArchive(srcBackend, dstBackend)
+	imported, failed, err := importArchive(srcBackend, dstBackend, srcConfig)
 	if err != nil {
 		t.Errorf("Import failed: %s", err.Error())
 	}
@@ -183,7 +183,7 @@ func TestImportDataIntegrity(t *testing.T) {
 	}
 
 	// Perform import
-	_, _, err = importArchive(srcBackend, dstBackend)
+	_, _, err = importArchive(srcBackend, dstBackend, srcConfig)
 	if err != nil {
 		t.Errorf("Import failed: %s", err.Error())
 	}
@@ -253,13 +253,13 @@ func TestImportEmptyArchive(t *testing.T) {
 	dstDb := filepath.Join(tmpdir, "dst-archive.db")
 
 	// Create empty source archive
-	if err := os.MkdirAll(srcArchive, 0755); err != nil {
+	if err := os.MkdirAll(srcArchive, 0o755); err != nil {
 		t.Fatalf("Failed to create source directory: %s", err.Error())
 	}
 
 	// Write version file
 	versionFile := filepath.Join(srcArchive, "version.txt")
-	if err := os.WriteFile(versionFile, []byte("3"), 0644); err != nil {
+	if err := os.WriteFile(versionFile, []byte("3"), 0o644); err != nil {
 		t.Fatalf("Failed to write version file: %s", err.Error())
 	}
 
@@ -277,7 +277,7 @@ func TestImportEmptyArchive(t *testing.T) {
 	}
 
 	// Perform import
-	imported, failed, err := importArchive(srcBackend, dstBackend)
+	imported, failed, err := importArchive(srcBackend, dstBackend, srcConfig)
 	if err != nil {
 		t.Errorf("Import from empty archive should not fail: %s", err.Error())
 	}
@@ -321,13 +321,13 @@ func TestImportDuplicateJobs(t *testing.T) {
 	}
 
 	// First import
-	imported1, _, err := importArchive(srcBackend, dstBackend)
+	imported1, _, err := importArchive(srcBackend, dstBackend, srcConfig)
 	if err != nil {
 		t.Fatalf("First import failed: %s", err.Error())
 	}
 
 	// Second import (should skip all jobs)
-	imported2, _, err := importArchive(srcBackend, dstBackend)
+	imported2, _, err := importArchive(srcBackend, dstBackend, srcConfig)
 	if err != nil {
 		t.Errorf("Second import failed: %s", err.Error())
 	}
@@ -366,7 +366,7 @@ func TestImportToEmptyFileDestination(t *testing.T) {
 	util.CopyDir(testDataPath, srcArchive)
 
 	// Setup empty destination directory
-	os.MkdirAll(dstArchive, 0755)
+	os.MkdirAll(dstArchive, 0o755)
 	// NOTE: NOT writing version.txt here!
 
 	// Initialize source
@@ -384,7 +384,7 @@ func TestImportToEmptyFileDestination(t *testing.T) {
 	}
 
 	// Perform import
-	imported, _, err := importArchive(srcBackend, dstBackend)
+	imported, _, err := importArchive(srcBackend, dstBackend, srcConfig)
 	if err != nil {
 		t.Errorf("Import failed: %v", err)
 	}

tools/archive-manager/main.go

@@ -5,12 +5,20 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io/fs"
 	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"strings"
 	"sync"
 	"sync/atomic"
+	"syscall"
 	"time"
 
 	"github.com/ClusterCockpit/cc-backend/internal/config"
@@ -33,80 +41,289 @@ func parseDate(in string) int64 {
 	return 0
 }
 
+// parseArchivePath extracts the path from the source config JSON.
+func parseArchivePath(srcConfig string) (string, error) {
+	var config struct {
+		Kind string `json:"kind"`
+		Path string `json:"path"`
+	}
+	if err := json.Unmarshal([]byte(srcConfig), &config); err != nil {
+		return "", fmt.Errorf("failed to parse source config: %w", err)
+	}
+
+	if config.Path == "" {
+		return "", fmt.Errorf("no path found in source config")
+	}
+
+	return config.Path, nil
+}
+
+// countJobsNative counts jobs using native Go filepath.WalkDir.
+// This is used as a fallback when fd/fdfind is not available.
+func countJobsNative(archivePath string) (int, error) {
+	count := 0
+	err := filepath.WalkDir(archivePath, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return nil // Skip directories we can't access
+		}
+		if !d.IsDir() && d.Name() == "meta.json" {
+			count++
+		}
+		return nil
+	})
+
+	if err != nil {
+		return 0, fmt.Errorf("failed to walk directory: %w", err)
+	}
+
+	return count, nil
+}
+
+// countJobsWithFd counts jobs using the external fd command.
+func countJobsWithFd(fdPath, archivePath string) (int, error) {
+	fdCmd := exec.Command(fdPath, "meta.json", archivePath)
+	wcCmd := exec.Command("wc", "-l")
+
+	pipe, err := fdCmd.StdoutPipe()
+	if err != nil {
+		return 0, fmt.Errorf("failed to create pipe: %w", err)
+	}
+	wcCmd.Stdin = pipe
+
+	if err := fdCmd.Start(); err != nil {
+		return 0, fmt.Errorf("failed to start fd command: %w", err)
+	}
+
+	output, err := wcCmd.Output()
+	if err != nil {
+		return 0, fmt.Errorf("failed to run wc command: %w", err)
+	}
+
+	if err := fdCmd.Wait(); err != nil {
+		return 0, fmt.Errorf("fd command failed: %w", err)
+	}
+
+	countStr := strings.TrimSpace(string(output))
+	count, err := strconv.Atoi(countStr)
+	if err != nil {
+		return 0, fmt.Errorf("failed to parse count from wc output '%s': %w", countStr, err)
+	}
+
+	return count, nil
+}
+
+// countJobs counts the total number of jobs in the source archive.
+// It tries to use external fd/fdfind command for speed, falling back to
+// native Go filepath.WalkDir if neither is available.
+// The srcConfig parameter should be the JSON configuration string containing the archive path.
+func countJobs(srcConfig string) (int, error) {
+	archivePath, err := parseArchivePath(srcConfig)
+	if err != nil {
+		return 0, err
+	}
+
+	// Try fd first (common name)
+	if fdPath, err := exec.LookPath("fd"); err == nil {
+		return countJobsWithFd(fdPath, archivePath)
+	}
+
+	// Try fdfind (Debian/Ubuntu package name)
+	if fdPath, err := exec.LookPath("fdfind"); err == nil {
+		return countJobsWithFd(fdPath, archivePath)
+	}
+
+	// Fall back to native Go implementation
+	cclog.Debug("fd/fdfind not found, using native Go file walker")
+	return countJobsNative(archivePath)
+}
+
+// formatDuration formats a duration as a human-readable string.
+func formatDuration(d time.Duration) string {
+	if d < time.Minute {
+		return fmt.Sprintf("%ds", int(d.Seconds()))
+	} else if d < time.Hour {
+		return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60)
+	}
+	return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60)
+}
+
+// progressMeter displays import progress to the terminal.
+type progressMeter struct {
+	total     int
+	processed int32
+	imported  int32
+	skipped   int32
+	failed    int32
+	startTime time.Time
+	done      chan struct{}
+}
+
+func newProgressMeter(total int) *progressMeter {
+	return &progressMeter{
+		total:     total,
+		startTime: time.Now(),
+		done:      make(chan struct{}),
+	}
+}
+
+func (p *progressMeter) start() {
+	go func() {
+		ticker := time.NewTicker(500 * time.Millisecond)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+				p.render()
+			case <-p.done:
+				p.render()
+				fmt.Println()
+				return
+			}
+		}
+	}()
+}
+
+func (p *progressMeter) render() {
+	processed := atomic.LoadInt32(&p.processed)
+	imported := atomic.LoadInt32(&p.imported)
+	skipped := atomic.LoadInt32(&p.skipped)
+	failed := atomic.LoadInt32(&p.failed)
+
+	elapsed := time.Since(p.startTime)
+	percent := float64(processed) / float64(p.total) * 100
+	if p.total == 0 {
+		percent = 0
+	}
+
+	var eta string
+	var throughput float64
+	if processed > 0 {
+		throughput = float64(processed) / elapsed.Seconds()
+		remaining := float64(p.total-int(processed)) / throughput
+		eta = formatDuration(time.Duration(remaining) * time.Second)
+	} else {
+		eta = "calculating..."
+	}
+
+	barWidth := 30
+	filled := int(float64(barWidth) * float64(processed) / float64(p.total))
+	if p.total == 0 {
+		filled = 0
+	}
+
+	var bar strings.Builder
+	for i := range barWidth {
+		if i < filled {
+			bar.WriteString("█")
+		} else {
+			bar.WriteString("░")
+		}
+	}
+
+	fmt.Printf("\r[%s] %5.1f%% | %d/%d | %.1f jobs/s | ETA: %s | ✓%d ○%d ✗%d   ",
+		bar.String(), percent, processed, p.total, throughput, eta, imported, skipped, failed)
+}
+
+func (p *progressMeter) stop() {
+	close(p.done)
+}
+
 // importArchive imports all jobs from a source archive backend to a destination archive backend.
 // It uses parallel processing with a worker pool to improve performance.
+// The import can be interrupted by CTRL-C (SIGINT) and will terminate gracefully.
 // Returns the number of successfully imported jobs, failed jobs, and any error encountered.
-func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, error) {
+func importArchive(srcBackend, dstBackend archive.ArchiveBackend, srcConfig string) (int, int, error) {
 	cclog.Info("Starting parallel archive import...")
+	cclog.Info("Press CTRL-C to interrupt (will finish current jobs before exiting)")
 
-	// Use atomic counters for thread-safe updates
+	ctx, cancel := context.WithCancel(context.Background())
-	var imported int32
+	defer cancel()
-	var failed int32
+
-	var skipped int32
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
+
+	var interrupted atomic.Bool
+
+	go func() {
+		<-sigChan
+		cclog.Warn("Interrupt received, stopping import (finishing current jobs)...")
+		interrupted.Store(true)
+		cancel()
+		// Stop listening for further signals to allow force quit with second CTRL-C
+		signal.Stop(sigChan)
+	}()
+
+	cclog.Info("Counting jobs in source archive (this may take a long time) ...")
+	totalJobs, err := countJobs(srcConfig)
+	if err != nil {
+		return 0, 0, fmt.Errorf("failed to count jobs: %w", err)
+	}
+	cclog.Infof("Found %d jobs to process", totalJobs)
+
+	progress := newProgressMeter(totalJobs)
 
-	// Number of parallel workers
 	numWorkers := 4
 	cclog.Infof("Using %d parallel workers", numWorkers)
 
-	// Create channels for job distribution
 	jobs := make(chan archive.JobContainer, numWorkers*2)
 
-	// WaitGroup to track worker completion
 	var wg sync.WaitGroup
 
-	// Start worker goroutines
+	progress.start()
+
 	for i := range numWorkers {
 		wg.Add(1)
 		go func(workerID int) {
 			defer wg.Done()
 
 			for job := range jobs {
-				// Validate job metadata
 				if job.Meta == nil {
 					cclog.Warn("Skipping job with nil metadata")
-					atomic.AddInt32(&failed, 1)
+					atomic.AddInt32(&progress.failed, 1)
+					atomic.AddInt32(&progress.processed, 1)
 					continue
 				}
 
-				// Validate job data
 				if job.Data == nil {
 					cclog.Warnf("Job %d from cluster %s has no metric data, skipping",
 						job.Meta.JobID, job.Meta.Cluster)
-					atomic.AddInt32(&failed, 1)
+					atomic.AddInt32(&progress.failed, 1)
+					atomic.AddInt32(&progress.processed, 1)
 					continue
 				}
 
-				// Check if job already exists in destination
 				if dstBackend.Exists(job.Meta) {
 					cclog.Debugf("Job %d (cluster: %s, start: %d) already exists in destination, skipping",
 						job.Meta.JobID, job.Meta.Cluster, job.Meta.StartTime)
-					atomic.AddInt32(&skipped, 1)
+					atomic.AddInt32(&progress.skipped, 1)
+					atomic.AddInt32(&progress.processed, 1)
 					continue
 				}
 
-				// Import job to destination
 				if err := dstBackend.ImportJob(job.Meta, job.Data); err != nil {
 					cclog.Errorf("Failed to import job %d from cluster %s: %s",
 						job.Meta.JobID, job.Meta.Cluster, err.Error())
-					atomic.AddInt32(&failed, 1)
+					atomic.AddInt32(&progress.failed, 1)
+					atomic.AddInt32(&progress.processed, 1)
 					continue
 				}
 
-				// Successfully imported
+				atomic.AddInt32(&progress.imported, 1)
-				newCount := atomic.AddInt32(&imported, 1)
+				atomic.AddInt32(&progress.processed, 1)
-				if newCount%100 == 0 {
-					cclog.Infof("Progress: %d jobs imported, %d skipped, %d failed",
-						newCount, atomic.LoadInt32(&skipped), atomic.LoadInt32(&failed))
-				}
 			}
 		}(i)
 	}
 
-	// Feed jobs to workers
 	go func() {
-		// Import cluster configs first
+		defer close(jobs)
+
 		clusters := srcBackend.GetClusters()
 		for _, clusterName := range clusters {
+			if ctx.Err() != nil {
+				return
+			}
+
 			clusterCfg, err := srcBackend.LoadClusterCfg(clusterName)
 			if err != nil {
 				cclog.Errorf("Failed to load cluster config for %s: %v", clusterName, err)
@@ -121,20 +338,33 @@ func importArchive(srcBackend, dstBackend archive.ArchiveBackend) (int, int, err
 		}
 
 		for job := range srcBackend.Iter(true) {
-			jobs <- job
+			select {
+			case <-ctx.Done():
+				// Drain remaining items from iterator to avoid resource leak
+				// but don't process them
+				return
+			case jobs <- job:
+			}
 		}
-		close(jobs)
 	}()
 
-	// Wait for all workers to complete
 	wg.Wait()
+	progress.stop()
 
-	finalImported := int(atomic.LoadInt32(&imported))
+	finalImported := int(atomic.LoadInt32(&progress.imported))
-	finalFailed := int(atomic.LoadInt32(&failed))
+	finalFailed := int(atomic.LoadInt32(&progress.failed))
-	finalSkipped := int(atomic.LoadInt32(&skipped))
+	finalSkipped := int(atomic.LoadInt32(&progress.skipped))
 
-	cclog.Infof("Import completed: %d jobs imported, %d skipped, %d failed",
+	elapsed := time.Since(progress.startTime)
-		finalImported, finalSkipped, finalFailed)
+
+	if interrupted.Load() {
+		cclog.Warnf("Import interrupted after %s: %d jobs imported, %d skipped, %d failed",
+			formatDuration(elapsed), finalImported, finalSkipped, finalFailed)
+		return finalImported, finalFailed, fmt.Errorf("import interrupted by user")
+	}
+
+	cclog.Infof("Import completed in %s: %d jobs imported, %d skipped, %d failed",
+		formatDuration(elapsed), finalImported, finalSkipped, finalFailed)
 
 	if finalFailed > 0 {
 		return finalImported, finalFailed, fmt.Errorf("%d jobs failed to import", finalFailed)
@@ -150,7 +380,7 @@ func main() {
 
 	flag.StringVar(&srcPath, "s", "./var/job-archive", "Specify the source job archive path. Default is ./var/job-archive")
 	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
-	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
+	flag.StringVar(&flagLogLevel, "loglevel", "info", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
 	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
 	flag.StringVar(&flagRemoveCluster, "remove-cluster", "", "Remove cluster from archive and database")
 	flag.StringVar(&flagRemoveBefore, "remove-before", "", "Remove all jobs with start time before date (Format: 2006-Jan-04)")
@@ -188,7 +418,7 @@ func main() {
 		cclog.Info("Destination backend initialized successfully")
 
 		// Perform import
-		imported, failed, err := importArchive(srcBackend, dstBackend)
+		imported, failed, err := importArchive(srcBackend, dstBackend, flagSrcConfig)
 		if err != nil {
 			cclog.Errorf("Import completed with errors: %s", err.Error())
 			if failed > 0 {

web/frontend/rollup.config.mjs

@@ -74,5 +74,6 @@ export default [
     entrypoint('node', 'src/node.entrypoint.js'),
     entrypoint('analysis', 'src/analysis.entrypoint.js'),
     entrypoint('status', 'src/status.entrypoint.js'),
+    entrypoint('dashpublic', 'src/dashpublic.entrypoint.js'),
     entrypoint('config', 'src/config.entrypoint.js')
 ];

web/frontend/src/Config.root.svelte

@@ -7,6 +7,7 @@
   - `isApi Bool!`: Is currently logged in user api authority
   - `username String!`: Empty string if auth. is disabled, otherwise the username as string
   - `ncontent String!`: The currently displayed message on the homescreen
+  - `clusters [String]`: The available clusternames
 -->
 
 <script>
@@ -22,6 +23,7 @@
     isApi,
     username,
     ncontent,
+    clusters
   } = $props();
 </script>
 
@@ -30,7 +32,7 @@
     <CardHeader>
       <CardTitle class="mb-1">Admin Options</CardTitle>
     </CardHeader>
-    <AdminSettings {ncontent}/>
+    <AdminSettings {ncontent} {clusters}/>
   </Card>
 {/if}
 

web/frontend/src/DashPublic.root.svelte

@@ -0,0 +1,579 @@
+<!--
+  @component Main cluster status view component; renders current system-usage information
+
+  Properties:
+  - `presetCluster String`: The cluster to show status information for
+-->
+
+ <script>
+  // import {
+  //   getContext
+  // } from "svelte"
+  import {
+    queryStore,
+    gql,
+    getContextClient,
+  } from "@urql/svelte";
+  import {
+    init,
+  } from "./generic/utils.js";
+  import {
+    formatNumber,
+  } from "./generic/units.js";
+  import {
+    Row,
+    Col,
+    Card,
+    CardHeader,
+    CardBody,
+    Spinner,
+    Table,
+    Progress,
+    Icon,
+    Button
+  } from "@sveltestrap/sveltestrap";
+  import Roofline from "./generic/plots/Roofline.svelte";
+  import Pie, { colors } from "./generic/plots/Pie.svelte";
+  import Stacked from "./generic/plots/Stacked.svelte";
+  import DoubleMetric from "./generic/plots/DoubleMetricPlot.svelte";
+  import Refresher from "./generic/helper/Refresher.svelte";
+
+  /* Svelte 5 Props */
+  let {
+    presetCluster,
+  } = $props();
+
+  /*Const Init */
+  const { query: initq } = init();
+  const client = getContextClient();
+  // const useCbColors = getContext("cc-config")?.plotConfiguration_colorblindMode || false
+
+  /* States */
+  let from = $state(new Date(Date.now() - (5 * 60 * 1000)));
+  let clusterFrom = $state(new Date(Date.now() - (8 * 60 * 60 * 1000)));
+  let to = $state(new Date(Date.now()));
+  let stackedFrom = $state(Math.floor(Date.now() / 1000) - 14400);
+  let colWidthStates = $state(0);
+  let colWidthRoof = $state(0);
+  let colWidthTotals = $state(0);
+  let colWidthStacked = $state(0);
+
+  /* Derived */
+  // States for Stacked charts
+  const statesTimed = $derived(queryStore({
+    client: client,
+    query: gql`
+      query ($filter: [NodeFilter!], $type: String!) {
+        nodeStatesTimed(filter: $filter, type: $type) {
+          state
+          counts
+          times
+        }
+      }
+    `,
+    variables: {
+      filter: { cluster: { eq: presetCluster }, timeStart: stackedFrom},
+      type: "node",
+    },
+    requestPolicy: "network-only"
+  }));
+
+  // Note: nodeMetrics are requested on configured $timestep resolution
+  // Result: The latest 5 minutes (datapoints) for each node independent of job
+  const statusQuery = $derived(queryStore({
+    client: client,
+    query: gql`
+      query (
+        $cluster: String!
+        $metrics: [String!]
+        $from: Time!
+        $to: Time!
+        $clusterFrom: Time!
+        $jobFilter: [JobFilter!]!
+        $nodeFilter: [NodeFilter!]!
+        $paging: PageRequest!
+        $sorting: OrderByInput!
+      ) {
+        # Node 5 Minute Averages for Roofline
+        nodeMetrics(
+          cluster: $cluster
+          metrics: $metrics
+          from: $from
+          to: $to
+        ) {
+          host
+          subCluster
+          metrics {
+            name
+            metric {
+              series {
+                statistics {
+                  avg
+                }
+              }
+            }
+          }
+        }
+        # Running Job Metric Average for Rooflines
+        jobsMetricStats(filter: $jobFilter, metrics: $metrics) {
+          id
+          jobId
+          duration
+          numNodes
+          numAccelerators
+          subCluster
+          stats {
+            name
+            data {
+              avg
+            }
+          }
+        }
+        # Get Jobs for Per-Node Counts
+        jobs(filter: $jobFilter, order: $sorting, page: $paging) {
+          items {
+            jobId
+            resources {
+              hostname
+            }
+          }
+          count
+        }
+        # Only counts shared nodes once 
+        allocatedNodes(cluster: $cluster) {
+          name
+          count
+        }
+        # Get Current States fir Pie Charts
+        nodeStates(filter: $nodeFilter) {
+          state
+          count
+        }
+        # Get States for Node Roofline; $sorting unused in backend: Use placeholder
+        nodes(filter: $nodeFilter, order: $sorting) {
+          count
+          items {
+            hostname
+            cluster
+            subCluster
+            schedulerState
+          }
+        }
+        # totalNodes includes multiples if shared jobs: Info-Card Data
+        jobsStatistics(
+          filter: $jobFilter
+          page: $paging
+          sortBy: TOTALJOBS
+          groupBy: SUBCLUSTER
+        ) {
+          id
+          totalJobs
+          totalUsers
+          totalCores
+          totalAccs
+        }
+        # ClusterMetrics for doubleMetricPlot
+        clusterMetrics(
+          cluster: $cluster
+          metrics: $metrics
+          from: $clusterFrom
+          to: $to
+        ) {
+          nodeCount
+          metrics {
+            name
+            unit {
+              prefix
+              base
+            }
+            timestep
+            data
+          }
+        }
+      }
+    `,
+    variables: {
+      cluster: presetCluster,
+      metrics: ["flops_any", "mem_bw"], // Metrics For Cluster Plot and Roofline
+      from: from.toISOString(),
+      clusterFrom: clusterFrom.toISOString(),
+      to: to.toISOString(),
+      jobFilter: [{ state: ["running"] }, { cluster: { eq: presetCluster } }],
+      nodeFilter: { cluster: { eq: presetCluster }},
+      paging: { itemsPerPage: -1, page: 1 }, // Get all: -1
+      sorting: { field: "startTime", type: "col", order: "DESC" }
+    },
+    requestPolicy: "network-only"
+  }));
+
+  const clusterInfo = $derived.by(() => {
+    if ($initq?.data?.clusters) {
+      let rawInfos = {};
+      let subClusters = $initq?.data?.clusters?.find((c) => c.name == presetCluster)?.subClusters || [];
+      for (let subCluster of subClusters) {
+        // Allocations
+        if (!rawInfos['allocatedNodes']) rawInfos['allocatedNodes'] = $statusQuery?.data?.allocatedNodes?.find(({ name }) => name == subCluster.name)?.count || 0;
+        else rawInfos['allocatedNodes'] += $statusQuery?.data?.allocatedNodes?.find(({ name }) => name == subCluster.name)?.count || 0;
+
+        if (!rawInfos['allocatedCores']) rawInfos['allocatedCores'] = $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalCores || 0;
+        else rawInfos['allocatedCores'] += $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalCores || 0;
+
+        if (!rawInfos['allocatedAccs']) rawInfos['allocatedAccs'] = $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalAccs || 0;
+        else rawInfos['allocatedAccs'] += $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalAccs || 0;
+
+        // Infos
+        if (!rawInfos['processorTypes']) rawInfos['processorTypes'] = subCluster?.processorType ? new Set([subCluster.processorType]) : new Set([]);
+        else rawInfos['processorTypes'].add(subCluster.processorType);
+
+        if (!rawInfos['activeUsers']) rawInfos['activeUsers'] = $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalUsers || 0;
+        else rawInfos['activeUsers'] += $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalUsers || 0;
+
+        if (!rawInfos['runningJobs']) rawInfos['runningJobs'] = $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalJobs || 0;
+        else rawInfos['runningJobs'] += $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalJobs || 0;
+
+        if (!rawInfos['totalNodes']) rawInfos['totalNodes'] = subCluster?.numberOfNodes || 0;
+        else rawInfos['totalNodes'] += subCluster?.numberOfNodes || 0;
+
+        if (!rawInfos['totalCores']) rawInfos['totalCores'] = (subCluster?.socketsPerNode * subCluster?.coresPerSocket * subCluster?.numberOfNodes) || 0;
+        else rawInfos['totalCores'] += (subCluster?.socketsPerNode * subCluster?.coresPerSocket * subCluster?.numberOfNodes) || 0;
+
+        if (!rawInfos['totalAccs']) rawInfos['totalAccs'] = (subCluster?.numberOfNodes * subCluster?.topology?.accelerators?.length) || 0;
+        else rawInfos['totalAccs'] += (subCluster?.numberOfNodes * subCluster?.topology?.accelerators?.length) || 0;
+          
+        // Units (Set Once)
+        if (!rawInfos['flopRateUnit']) rawInfos['flopRateUnit'] = subCluster.flopRateSimd.unit.prefix + subCluster.flopRateSimd.unit.base
+        if (!rawInfos['memBwRateUnit']) rawInfos['memBwRateUnit'] = subCluster.memoryBandwidth.unit.prefix + subCluster.memoryBandwidth.unit.base
+
+        // Get Maxima For Roofline Knee Render
+        if (!rawInfos['roofData']) {
+          rawInfos['roofData'] = {
+            flopRateScalar: {value: subCluster.flopRateScalar.value},
+            flopRateSimd: {value: subCluster.flopRateSimd.value},
+            memoryBandwidth: {value: subCluster.memoryBandwidth.value}
+          };
+        } else { 
+          rawInfos['roofData']['flopRateScalar']['value'] = Math.max(rawInfos['roofData']['flopRateScalar']['value'], subCluster.flopRateScalar.value)
+          rawInfos['roofData']['flopRateSimd']['value'] = Math.max(rawInfos['roofData']['flopRateSimd']['value'], subCluster.flopRateSimd.value)
+          rawInfos['roofData']['memoryBandwidth']['value'] = Math.max(rawInfos['roofData']['memoryBandwidth']['value'], subCluster.memoryBandwidth.value)
+        }
+      }
+
+      // Keymetrics (Data on Cluster-Scope)
+      let rawFlops = $statusQuery?.data?.nodeMetrics?.reduce((sum, node) =>
+        sum + (node.metrics.find((m) => m.name == 'flops_any')?.metric?.series[0]?.statistics?.avg || 0),
+        0, // Initial Value
+      ) || 0;
+      rawInfos['flopRate'] = Math.floor((rawFlops * 100) / 100)
+
+      let rawMemBw = $statusQuery?.data?.nodeMetrics?.reduce((sum, node) =>
+        sum + (node.metrics.find((m) => m.name == 'mem_bw')?.metric?.series[0]?.statistics?.avg || 0),
+        0, // Initial Value
+      ) || 0;
+      rawInfos['memBwRate'] = Math.floor((rawMemBw * 100) / 100)
+
+      return rawInfos
+    } else {
+      return {};
+    }
+  });
+
+  const refinedStateData = $derived.by(() => {
+    return $statusQuery?.data?.nodeStates.
+      filter((e) => ['allocated', 'reserved', 'idle', 'mixed','down', 'unknown'].includes(e.state)).
+      sort((a, b) => b.count - a.count)
+  });
+
+  /* Functions */
+  function transformNodesStatsToData(subclusterData) {
+    let data = null
+    const x = [], y = []
+
+    if (subclusterData) {
+      for (let i = 0; i < subclusterData.length; i++) {
+        const flopsData = subclusterData[i].metrics.find((s) => s.name == "flops_any")
+        const memBwData = subclusterData[i].metrics.find((s) => s.name == "mem_bw")
+
+        const f = flopsData.metric.series[0].statistics.avg
+        const m = memBwData.metric.series[0].statistics.avg
+
+        let intensity = f / m
+        if (Number.isNaN(intensity) || !Number.isFinite(intensity)) {
+            intensity = 0.0 // Set to Float Zero: Will not show in Log-Plot (Always below render limit)
+        }
+
+        x.push(intensity)
+        y.push(f)
+      }
+    } else {
+        // console.warn("transformNodesStatsToData: metrics for 'mem_bw' and/or 'flops_any' missing!")
+    }
+
+    if (x.length > 0 && y.length > 0) {
+        data = [null, [x, y]] // for dataformat see roofline.svelte
+    }
+    return data
+  }
+
+  function transformNodesStatsToInfo(subClusterData) {
+    let result = [];
+    if (subClusterData) { //  && $nodesState?.data) {
+      // Use Nodes as Returned from CCMS, *NOT* as saved in DB via SlurmState-API!
+      for (let j = 0; j < subClusterData.length; j++) {
+        const nodeName = subClusterData[j]?.host ? subClusterData[j].host : "unknown"
+        const nodeMatch = $statusQuery?.data?.nodes?.items?.find((n) => n.hostname == nodeName && n.subCluster == subClusterData[j].subCluster);
+        const schedulerState = nodeMatch?.schedulerState ? nodeMatch.schedulerState : "notindb"
+        let numJobs = 0
+
+        if ($statusQuery?.data) {
+          const nodeJobs = $statusQuery?.data?.jobs?.items?.filter((job) => job.resources.find((res) => res.hostname == nodeName))
+          numJobs = nodeJobs?.length ? nodeJobs.length : 0
+        }
+
+        result.push({nodeName: nodeName, schedulerState: schedulerState, numJobs: numJobs})
+      };
+    };
+    return result
+  }
+
+</script>
+
+<Card style="height: 98vh;">
+  <CardBody class="align-content-center">
+    <Row>
+      <Col>
+        <Refresher
+          hideSelector
+          initially={60}
+          onRefresh={(interval) => {
+            from = new Date(Date.now() - 5 * 60 * 1000);
+            to = new Date(Date.now());
+            clusterFrom = new Date(Date.now() - (8 * 60 * 60 * 1000))
+
+            if (interval) stackedFrom += Math.floor(interval / 1000);
+            else stackedFrom += 1 // Workaround: TimeSelection not linked, just trigger new data on manual refresh
+          }}
+        />
+      </Col>
+      <Col class="d-flex justify-content-end">
+        <Button outline class="mb-1" size="sm" color="light" href="/">
+          <Icon name="x"/>
+        </Button>
+      </Col>
+    </Row>
+    {#if $statusQuery.fetching || $statesTimed.fetching}
+      <Row class="justify-content-center">
+        <Col xs="auto">
+          <Spinner />
+        </Col>
+      </Row>
+
+    {:else if $statusQuery.error || $statesTimed.error}
+      <Row cols={{xs:1, md:2}}>
+        {#if $statusQuery.error}
+          <Col>
+            <Card color="danger"><CardBody>Error Requesting Status Data: {$statusQuery.error.message}</CardBody></Card>
+          </Col>
+        {/if}
+        {#if $statesTimed.error}
+          <Col>
+            <Card color="danger"><CardBody>Error Requesting Node Scheduler States: {$statesTimed.error.message}</CardBody></Card>
+          </Col>
+        {/if}
+      </Row>
+
+    {:else}
+      <Row cols={{xs:1, md:2}}>
+        <Col> <!-- General Cluster Info Card -->
+           <Card class="h-100">
+            <CardHeader class="text-center">
+              <h2 class="mb-0">Cluster {presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}</h2>
+            </CardHeader>
+            <CardBody>
+              <h4>CPU(s)</h4><p><strong>{[...clusterInfo?.processorTypes].join(', ')}</strong></p>
+            </CardBody>
+           </Card>
+        </Col>
+
+        <Col> <!-- Utilization Info Card -->
+          <Card class="h-100">
+            <CardBody>
+              <Table borderless>
+                <tr class="py-2">
+                  <td style="font-size:x-large;">{clusterInfo?.runningJobs} Running Jobs</td>
+                  <td colspan="2" style="font-size:x-large;">{clusterInfo?.activeUsers} Active Users</td>
+                </tr>
+                <hr class="my-1"/>
+                <tr class="pt-2">
+                  <td style="font-size: large;">
+                    Flop Rate (<span style="cursor: help;" title="Flops[Any] = (Flops[Double] x 2) + Flops[Single]">Any</span>)
+                  </td>
+                  <td colspan="2" style="font-size: large;">
+                    Memory BW Rate
+                  </td>
+                </tr>
+                <tr class="pb-2">
+                  <td style="font-size:x-large;">
+                    {clusterInfo?.flopRate} 
+                    {clusterInfo?.flopRateUnit}
+                  </td>
+                  <td colspan="2" style="font-size:x-large;">
+                    {clusterInfo?.memBwRate} 
+                    {clusterInfo?.memBwRateUnit}
+                  </td>
+                </tr>
+                <hr class="my-1"/>
+                <tr class="py-2">
+                  <th scope="col">Allocated Nodes</th>
+                  <td style="min-width: 100px;"
+                    ><div class="col">
+                      <Progress
+                        value={clusterInfo?.allocatedNodes}
+                        max={clusterInfo?.totalNodes}
+                      />
+                    </div></td
+                  >
+                  <td
+                    >{clusterInfo?.allocatedNodes} / {clusterInfo?.totalNodes}
+                    Nodes</td
+                  >
+                </tr>
+                <tr class="py-2">
+                  <th scope="col">Allocated Cores</th>
+                  <td style="min-width: 100px;"
+                    ><div class="col">
+                      <Progress
+                        value={clusterInfo?.allocatedCores}
+                        max={clusterInfo?.totalCores}
+                      />
+                    </div></td
+                  >
+                  <td
+                    >{formatNumber(clusterInfo?.allocatedCores)} / {formatNumber(clusterInfo?.totalCores)}
+                    Cores</td
+                  >
+                </tr>
+                {#if clusterInfo?.totalAccs !== 0}
+                  <tr class="py-2">
+                    <th scope="col">Allocated Accelerators</th>
+                    <td style="min-width: 100px;"
+                      ><div class="col">
+                        <Progress
+                          value={clusterInfo?.allocatedAccs}
+                          max={clusterInfo?.totalAccs}
+                        />
+                      </div></td
+                    >
+                    <td
+                      >{clusterInfo?.allocatedAccs} / {clusterInfo?.totalAccs}
+                      Accelerators</td
+                    >
+                  </tr>
+                {/if}
+              </Table>
+            </CardBody>
+          </Card>
+        </Col>
+
+        <Col> <!-- Total Cluster Metric in Time SUMS-->
+          <div bind:clientWidth={colWidthTotals}>
+            <DoubleMetric
+              width={colWidthTotals}
+              timestep={$statusQuery?.data?.clusterMetrics[0]?.timestep || 60}
+              numNodes={$statusQuery?.data?.clusterMetrics?.nodeCount || 0}
+              metricData={$statusQuery?.data?.clusterMetrics?.metrics || []}
+              cluster={presetCluster}
+              fixLinewidth={2}
+            />
+          </div>
+        </Col>
+
+        <Col> <!-- Nodes Roofline -->
+          <div bind:clientWidth={colWidthRoof}>
+            {#key $statusQuery?.data?.nodeMetrics}
+              <Roofline
+                colorBackground
+                useColors={false}
+                useLegend={false}
+                allowSizeChange
+                width={colWidthRoof - 10}
+                height={300}
+                cluster={presetCluster}
+                subCluster={clusterInfo?.roofData ? clusterInfo.roofData : null}
+                roofData={transformNodesStatsToData($statusQuery?.data?.nodeMetrics)}
+                nodesData={transformNodesStatsToInfo($statusQuery?.data?.nodeMetrics)}
+                fixTitle="Node Utilization"
+                yMinimum={1.0}
+              />
+            {/key}
+          </div>
+        </Col>
+        <Col> <!-- Pie Last States -->
+          <Row>
+            {#if refinedStateData.length > 0}
+              <Col class="px-3 mt-2 mt-lg-0">
+                <div bind:clientWidth={colWidthStates}>
+                  {#key refinedStateData}
+                    <Pie
+                      canvasId="hpcpie-slurm"
+                      size={colWidthStates * 0.66}
+                      sliceLabel="Nodes"
+                      quantities={refinedStateData.map(
+                        (sd) => sd.count,
+                      )}
+                      entities={refinedStateData.map(
+                        (sd) => sd.state,
+                      )}
+                      fixColors={refinedStateData.map(
+                        (sd) => colors['nodeStates'][sd.state],
+                      )}
+                    />
+                  {/key}
+                </div>
+              </Col>
+              <Col class="px-4 py-2">
+                {#key refinedStateData}
+                  <Table>
+                    <tr class="mb-2">
+                      <th></th>
+                      <th class="h4">State</th>
+                      <th class="h4">Count</th>
+                    </tr>
+                    {#each refinedStateData as sd, i}
+                      <tr>
+                        <td><Icon name="circle-fill" style="color: {colors['nodeStates'][sd.state]}; font-size: 30px;"/></td>
+                        <td class="h5">{sd.state.charAt(0).toUpperCase() + sd.state.slice(1)}</td>
+                        <td class="h5">{sd.count}</td>
+                      </tr>
+                    {/each}
+                  </Table>
+                {/key}
+              </Col>
+            {:else}
+              <Col>
+                <Card body color="warning" class="mx-4 my-2"
+                  >Cannot render state status: No state data returned for <code>Pie Chart</code></Card
+                >
+              </Col>
+            {/if}
+          </Row>
+        </Col>
+
+        <Col> <!-- Stacked SchedState -->
+          <div bind:clientWidth={colWidthStacked}>
+            {#key $statesTimed?.data?.nodeStatesTimed}
+              <Stacked
+                data={$statesTimed?.data?.nodeStatesTimed}
+                width={colWidthStacked * 0.95}
+                xlabel="Time"
+                ylabel="Nodes"
+                yunit = "#Count"
+                title = "Cluster Status"
+                stateType = "Node"
+              />
+            {/key}
+          </div>
+        </Col>
+      </Row>
+    {/if}
+  </CardBody>
+</Card>

web/frontend/src/Header.svelte

@@ -120,7 +120,7 @@
       href: "/monitoring/status/",
       icon: "clipboard-data",
       perCluster: true,
-      listOptions: false,
+      listOptions: true,
       menu: "Info",
     },
   ];

web/frontend/src/Status.root.svelte

@@ -3,80 +3,39 @@
 
   Properties:
   - `presetCluster String`: The cluster to show status information for
+  - `displayType String`: The type of status component to render
 -->
 
  <script>
-  import {
-    getContext
-  } from "svelte"
-  import {
-    init,
-  } from "./generic/utils.js";
   import {
     Row,
     Col,
     Card,
-    CardBody,
+    CardBody
-    TabContent,
-    TabPane,
-    Spinner
   } from "@sveltestrap/sveltestrap";
 
-  import StatusDash from "./status/StatusDash.svelte";
+  import DashDetails from "./status/DashDetails.svelte";
-  import UsageDash from "./status/UsageDash.svelte";
+  import DashInternal from "./status/DashInternal.svelte";
-  import StatisticsDash from "./status/StatisticsDash.svelte";
 
   /* Svelte 5 Props */
   let {
-    presetCluster
+    presetCluster,
+    displayType
   } = $props();
-
-  /*Const Init */
-  const { query: initq } = init();
-  const useCbColors = getContext("cc-config")?.plotConfiguration_colorblindMode || false
 </script>
 
-<!-- Loading indicator & Refresh -->
+{#if displayType === 'DETAILS'}
-
+  <DashDetails {presetCluster}/>
-<Row cols={1} class="mb-2">
+{:else if displayType === 'DASHBOARD'}
-  <Col>
+  <DashInternal {presetCluster}/>
-    <h3 class="mb-0">Current Status of Cluster "{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}"</h3>
-  </Col>
-</Row>
-
-
-{#if $initq.fetching}
-  <Row cols={1} class="text-center mt-3">
-    <Col>
-      <Spinner />
-    </Col>
-  </Row>
-{:else if $initq.error}
-  <Row cols={1} class="text-center mt-3">
-    <Col>  
-      <Card body color="danger">{$initq.error.message}</Card>
-    </Col>
-  </Row>
 {:else}
-  <Card class="overflow-auto" style="height: auto;">
+  <Row>
-    <TabContent>
+    <Col>
-      <TabPane tabId="status-dash" tab="Status" active>
+      <Card color="danger">
         <CardBody>
-          <StatusDash clusters={$initq.data.clusters} {presetCluster} {useCbColors} useAltColors></StatusDash>
+          Unknown DisplayType for Status View!
         </CardBody>
-      </TabPane>
+      </Card>
-
+    </Col>
-      <TabPane tabId="usage-dash" tab="Usage">
+  </Row>
-        <CardBody>
-          <UsageDash {presetCluster} {useCbColors}></UsageDash>
-        </CardBody>
-      </TabPane>
-      
-      <TabPane tabId="metric-dash" tab="Statistics">
-        <CardBody>
-          <StatisticsDash {presetCluster} {useCbColors}></StatisticsDash>
-        </CardBody>
-      </TabPane>
-    </TabContent>
-  </Card>
 {/if}

web/frontend/src/config.entrypoint.js

@@ -10,6 +10,7 @@ mount(Config, {
         isApi: isApi,
         username: username,
         ncontent: ncontent,
+        clusters: hClusters.map((c) => c.name) // Defined in Header Template
     },
     context: new Map([
             ['cc-config', clusterCockpitConfig],

web/frontend/src/config/AdminSettings.svelte

@@ -3,6 +3,7 @@
 
   Properties:
   - `ncontent String`: The homepage notice content
+  - `clusters [String]`: The available clusternames
 -->
 
 <script>
@@ -17,7 +18,8 @@
 
   /* Svelte 5 Props */
   let {
-    ncontent
+    ncontent,
+    clusters
   } = $props();
 
   /* Const Init*/
@@ -66,6 +68,6 @@
   <Col>
     <EditProject reloadUser={() => getUserList()} />
   </Col>
-  <Options config={ccconfig}/>
+  <Options config={ccconfig} {clusters}/>
   <NoticeEdit {ncontent}/>
 </Row>

web/frontend/src/config/admin/Options.svelte

@@ -1,10 +1,18 @@
 <!--
   @component Admin option select card
+
+    Properties:
+  - `clusters [String]`: The available clusternames
 -->
 
 <script>
   import { getContext, onMount } from "svelte";
-  import { Col, Card, CardBody, CardTitle } from "@sveltestrap/sveltestrap";
+  import { Row, Col, Card, CardBody, CardTitle, Button, Icon } from "@sveltestrap/sveltestrap";
+
+  /* Svelte 5 Props */
+  let {
+    clusters,
+  } = $props();
 
   /*Const Init */
   const resampleConfig = getContext("resampling");
@@ -44,6 +52,26 @@
   </Card>
 </Col>
 
+{#if clusters?.length > 0}
+  <Col> 
+    <Card class="h-100">
+      <CardBody>
+        <CardTitle class="mb-3">Public Dashboard Links</CardTitle>
+        <Row>
+        {#each clusters as cluster}
+          <Col>
+            <Button color="info" class="mb-2 mb-xl-0" href={`/monitoring/dashboard/${cluster}`} target="_blank">
+              <Icon name="clipboard-pulse" class="mr-2"/>
+              {cluster.charAt(0).toUpperCase() + cluster.slice(1)} Public Dashboard
+            </Button>
+          </Col>
+        {/each}
+        </Row>
+      </CardBody>
+    </Card>
+  </Col>
+{/if}
+
 {#if resampleConfig}
   <Col> 
     <Card class="h-100">

web/frontend/src/dashpublic.entrypoint.js

@@ -0,0 +1,13 @@
+import { mount } from 'svelte';
+// import {} from './header.entrypoint.js'
+import DashPublic from './DashPublic.root.svelte'
+
+mount(DashPublic, {
+    target: document.getElementById('svelte-app'),
+    props: {
+        presetCluster: presetCluster,
+    },
+    context: new Map([
+            ['cc-config', clusterCockpitConfig]
+    ])
+})

web/frontend/src/generic/helper/Refresher.svelte

@@ -14,6 +14,7 @@
   let {
     initially = null,
     presetClass = "",
+    hideSelector = false,
     onRefresh
   } = $props();
 
@@ -36,25 +37,27 @@
   });
 </script>
 
-<InputGroup class={presetClass}>
+{#if !hideSelector}
-  <Input
+  <InputGroup class={presetClass}>
-    type="select"
+    <Input
-    title="Periodic refresh interval"
+      type="select"
-    bind:value={refreshInterval}
+      title="Periodic refresh interval"
-    onchange={refreshIntervalChanged}
+      bind:value={refreshInterval}
-  >
+      onchange={refreshIntervalChanged}
-    <option value={null}>No Interval</option>
-    <option value={30 * 1000}>30 Seconds</option>
-    <option value={60 * 1000}>60 Seconds</option>
-    <option value={2 * 60 * 1000}>Two Minutes</option>
-    <option value={5 * 60 * 1000}>5 Minutes</option>
-  </Input>
-  <Button
-    outline
-    onclick={() => onRefresh(refreshInterval)}
-    disabled={refreshInterval != null}
     >
-    <Icon name="arrow-clockwise" /> Refresh
+      <option value={null}>No Interval</option>
-  </Button>
+      <option value={30 * 1000}>30 Seconds</option>
-</InputGroup>
+      <option value={60 * 1000}>60 Seconds</option>
+      <option value={2 * 60 * 1000}>Two Minutes</option>
+      <option value={5 * 60 * 1000}>5 Minutes</option>
+    </Input>
+    <Button
+      outline
+      onclick={() => onRefresh(refreshInterval)}
+      disabled={refreshInterval != null}
+      >
+      <Icon name="arrow-clockwise" /> Refresh
+    </Button>
+  </InputGroup>
+{/if}
 

web/frontend/src/generic/plots/DoubleMetricPlot.svelte

@@ -0,0 +1,642 @@
+<!--
+  @component Main plot component, based on uPlot; metricdata values by time
+
+  Only width/height should change reactively.
+
+  Properties:
+  - `metric String`: The metric name
+  - `scope String?`: Scope of the displayed data [Default: node]
+  - `height Number?`: The plot height [Default: 300]
+  - `timestep Number`: The timestep used for X-axis rendering
+  - `series [GraphQL.Series]`: The metric data object
+  - `statisticsSeries [GraphQL.StatisticsSeries]?`: Min/Max/Median representation of metric data [Default: null]
+  - `cluster String?`: Cluster name of the parent job / data [Default: ""]
+  - `subCluster String`: Name of the subCluster of the parent job
+  - `isShared Bool?`: If this job used shared resources; for additional legend display [Default: false]
+  - `forNode Bool?`: If this plot is used for node data display; will render x-axis as negative time with $now as maximum [Default: false]
+  - `numhwthreads Number?`: Number of job HWThreads [Default: 0]
+  - `numaccs Number?`: Number of job Accelerators [Default: 0]
+  - `zoomState Object?`: The last zoom state to preserve on user zoom [Default: null]
+  - `thersholdState Object?`: The last threshold state to preserve on user zoom [Default: null]
+  - `extendedLegendData Object?`: Additional information to be rendered in an extended legend [Default: null]
+  - `onZoom Func`: Callback function to handle zoom-in event
+-->
+
+<script>
+  import uPlot from "uplot";
+  import { formatNumber, formatDurationTime } from "../units.js";
+  import { getContext, onMount, onDestroy } from "svelte";
+  import { Card } from "@sveltestrap/sveltestrap";
+
+  /* Svelte 5 Props */
+  let {
+    // metric,
+    width = 0,
+    height = 300,
+    fixLinewidth = null,
+    timestep,
+    numNodes,
+    metricData,
+    // useStatsSeries = false,
+    // statisticsSeries = null,
+    cluster = "",
+    forNode = true,
+    // zoomState = null,
+    // thresholdState = null,
+    enableFlip = false,
+    // onZoom
+  } = $props();
+
+  /* Const Init */
+  const clusterCockpitConfig = getContext("cc-config");
+  // const resampleConfig = getContext("resampling");
+  // const subClusterTopology = getContext("getHardwareTopology")(cluster, subCluster);
+  // const metricConfig = getContext("getMetricConfig")(cluster, subCluster, metric);
+  const lineColors = clusterCockpitConfig.plotConfiguration_colorScheme;
+  const lineWidth = fixLinewidth ? fixLinewidth : clusterCockpitConfig.plotConfiguration_lineWidth / window.devicePixelRatio;
+  // const cbmode = clusterCockpitConfig?.plotConfiguration_colorblindMode || false;
+  const renderSleepTime = 200;
+  // const normalLineColor = "#000000";
+  // const backgroundColors = {
+  //   normal: "rgba(255, 255, 255, 1.0)",
+  //   caution: cbmode ? "rgba(239, 230, 69, 0.3)" : "rgba(255, 128, 0, 0.3)",
+  //   alert: cbmode ? "rgba(225, 86, 44, 0.3)" : "rgba(255, 0, 0, 0.3)",
+  // };
+
+  /* Var Init */
+  let timeoutId = null;
+
+  /* State Init */
+  let plotWrapper = $state(null);
+  let uplot = $state(null);
+
+  /* Derived */
+  // const usesMeanStatsSeries = $derived((statisticsSeries?.mean && statisticsSeries.mean.length != 0));
+  // const resampleTrigger = $derived(resampleConfig?.trigger ? Number(resampleConfig.trigger) : null);
+  // const resampleResolutions = $derived(resampleConfig?.resolutions ? [...resampleConfig.resolutions] : null);
+  // const resampleMinimum = $derived(resampleConfig?.resolutions ? Math.min(...resampleConfig.resolutions) : null);
+  // const thresholds = $derived(findJobAggregationThresholds(
+  //   subClusterTopology,
+  //   metricConfig,
+  //   scope,
+  //   numhwthreads,
+  //   numaccs
+  // ));
+  const longestSeries = $derived.by(() => {
+    // if (useStatsSeries) {
+    //   return usesMeanStatsSeries ? statisticsSeries?.mean?.length : statisticsSeries?.median?.length;
+    // } else {
+      return metricData.reduce((n, m) => Math.max(n, m.data.length), 0);
+    // }
+  });
+  const maxX = $derived(longestSeries * timestep);
+  // const maxY = $derived.by(() => {
+  //   let pendingY = 0;
+  //   // if (useStatsSeries) {
+  //   //   pendingY = statisticsSeries.max.reduce(
+  //   //     (max, x) => Math.max(max, x),
+  //   //     thresholds?.normal,
+  //   //   ) || thresholds?.normal
+  //   // } else { 
+  //     pendingY = series.reduce(
+  //         (max, series) => Math.max(max, series?.statistics?.max),
+  //         thresholds?.normal,
+  //       ) || thresholds?.normal;
+  //   // }
+
+  //   if (pendingY >= 10 * thresholds.peak) {
+  //     // Hard y-range render limit if outliers in series data
+  //     return (10 * thresholds.peak);
+  //   } else {
+  //     return pendingY;
+  //   }
+  // });
+  // const plotBands = $derived.by(() => {
+  //   if (useStatsSeries) {
+  //     return [
+  //       { series: [2, 3], fill: cbmode ? "rgba(0,0,255,0.1)" : "rgba(0,255,0,0.1)" },
+  //       { series: [3, 1], fill: cbmode ? "rgba(0,255,0,0.1)" : "rgba(255,0,0,0.1)" },
+  //     ];
+  //   };
+  //   return null;
+  // })
+  const plotData = $derived.by(() => {
+    let pendingData = [new Array(longestSeries)];
+    // X
+    if (forNode === true) {
+      // Negative Timestamp Buildup
+      for (let i = 0; i <= longestSeries; i++) {
+        pendingData[0][i] = (longestSeries - i) * timestep * -1;
+      }
+    } else {
+      // Positive Timestamp Buildup
+      for (let j = 0; j < longestSeries; j++) {
+        pendingData[0][j] = j * timestep;
+      };
+    };
+    // Y
+    // if (useStatsSeries) {
+    //   pendingData.push(statisticsSeries.min);
+    //   pendingData.push(statisticsSeries.max);
+    //   if (usesMeanStatsSeries) {
+    //     pendingData.push(statisticsSeries.mean);
+    //   } else {
+    //     pendingData.push(statisticsSeries.median);
+    //   }
+
+    // } else {
+      for (let i = 0; i < metricData.length; i++) {
+        pendingData.push(metricData[i]?.data);
+      };
+    // };
+    return pendingData;
+  })
+  const plotSeries = $derived.by(() => {
+    let pendingSeries = [
+      // Note: X-Legend Will not be shown as soon as Y-Axis are in extendedMode
+      {
+        label: "Runtime",
+        value: (u, ts, sidx, didx) =>
+        (didx == null) ? null : formatDurationTime(ts, forNode),
+      }
+    ];
+    // Y
+    // if (useStatsSeries) {
+    //   pendingSeries.push({
+    //     label: "min",
+    //     scale: "y",
+    //     width: lineWidth,
+    //     stroke: cbmode ? "rgb(0,255,0)" : "red",
+    //   });
+    //   pendingSeries.push({
+    //     label: "max",
+    //     scale: "y",
+    //     width: lineWidth,
+    //     stroke: cbmode ? "rgb(0,0,255)" : "green",
+    //   });
+    //   pendingSeries.push({
+    //     label: usesMeanStatsSeries ? "mean" : "median",
+    //     scale: "y",
+    //     width: lineWidth,
+    //     stroke: "black",
+    //   });
+
+    // } else {
+      for (let i = 0; i < metricData.length; i++) {
+        // Default
+        // if (!extendedLegendData) {
+          pendingSeries.push({
+            label: `${metricData[i]?.name} (${metricData[i]?.unit?.prefix}${metricData[i]?.unit?.base})`,
+            scale: `y${i+1}`,
+            width: lineWidth,
+            stroke: lineColor(i, metricData.length),
+          });
+        // }
+        // Extended Legend For NodeList
+      //   else {
+      //     pendingSeries.push({
+      //       label: 
+      //         scope === "node"
+      //           ? series[i].hostname
+      //           : scope === "accelerator"
+      //             ? 'Acc #' + (i + 1) // series[i].id.slice(9, 14) | Too Hardware Specific
+      //             : scope + " #" + (i + 1),
+      //       scale: "y",
+      //       width: lineWidth,
+      //       stroke: lineColor(i, series?.length),
+      //       values: (u, sidx, idx) => {
+      //         // "i" = "sidx - 1" : sidx contains x-axis-data
+      //         if (idx == null)
+      //           return {
+      //             time: '-',
+      //             value: '-',
+      //             user: '-',
+      //             job: '-'
+      //           };
+
+      //         if (series[i].id in extendedLegendData) {
+      //           return {
+      //             time: formatDurationTime(plotData[0][idx], forNode),
+      //             value: plotData[sidx][idx],
+      //             user: extendedLegendData[series[i].id].user,
+      //             job: extendedLegendData[series[i].id].job,
+      //           };
+      //         } else {
+      //           return {
+      //             time: formatDurationTime(plotData[0][idx], forNode),
+      //             value: plotData[sidx][idx],
+      //             user: '-',
+      //             job: '-',
+      //           };
+      //         }
+      //       }
+      //     });
+      //   }
+      // };
+    };
+    return pendingSeries;
+  })
+
+  /* Effects */
+  // $effect(() => {
+  //   if (!useStatsSeries && statisticsSeries != null) useStatsSeries = true;
+  // })
+
+  // This updates plot on all size changes if wrapper (== data) exists
+  $effect(() => {
+    if (plotWrapper) {
+      onSizeChange(width, height);
+    }
+  });
+
+  /* Functions */
+  function timeIncrs(timestep, maxX, forNode) {
+    if (forNode === true) {
+      return [60, 120, 240, 300, 360, 480, 600, 900, 1800, 3600, 7200, 14400, 21600]; // forNode fixed increments
+    } else {
+      let incrs = [];
+      for (let t = timestep; t < maxX; t *= 10)
+        incrs.push(t, t * 2, t * 3, t * 5);
+
+      return incrs;
+    }
+  }
+
+  // removed arg "subcluster": input metricconfig and topology now directly derived from subcluster
+  // function findJobAggregationThresholds(
+  //   subClusterTopology,
+  //   metricConfig,
+  //   scope,
+  //   numhwthreads,
+  //   numaccs
+  // ) {
+
+  //   if (!subClusterTopology || !metricConfig || !scope) {
+  //     console.warn("Argument missing for findJobAggregationThresholds!");
+  //     return null;
+  //   }
+
+  //   // handle special *-stat scopes
+  //   if (scope.match(/(.*)-stat$/)) {
+  //     const statParts = scope.split('-');
+  //     scope = statParts[0]
+  //   }
+
+  //   if (metricConfig?.aggregation == "avg") {
+  //     // Return as Configured
+  //     return {
+  //       normal: metricConfig.normal,
+  //       caution: metricConfig.caution,
+  //       alert: metricConfig.alert,
+  //       peak: metricConfig.peak,
+  //     };
+  //   }
+
+  //   if (metricConfig?.aggregation == "sum") {
+  //     // Scale Thresholds
+  //     let fraction;
+  //     if (numaccs > 0) fraction = subClusterTopology.accelerators.length / numaccs;
+  //     else if (numhwthreads > 0) fraction = subClusterTopology.core.length / numhwthreads;
+  //     else fraction = 1; // Fallback
+
+  //     let divisor;
+  //     // Exclusive: Fraction = 1; Shared: Fraction > 1
+  //     if (scope == 'node')              divisor = fraction;
+  //     // Cap divisor at number of available sockets or domains
+  //     else if (scope == 'socket')       divisor = (fraction < subClusterTopology.socket.length) ? subClusterTopology.socket.length : fraction;
+  //     else if (scope == "memoryDomain") divisor = (fraction < subClusterTopology.memoryDomain.length) ? subClusterTopology.socket.length : fraction;
+  //     // Use Maximum Division for Smallest Scopes
+  //     else if (scope == "core")         divisor = subClusterTopology.core.length;
+  //     else if (scope == "hwthread")     divisor = subClusterTopology.core.length; // alt. name for core
+  //     else if (scope == "accelerator")  divisor = subClusterTopology.accelerators.length;
+  //     else {
+  //       console.log('Unknown scope, return default aggregation thresholds for sum', scope)
+  //       divisor = 1;
+  //     }
+
+  //     return {
+  //       peak: metricConfig.peak / divisor,
+  //       normal: metricConfig.normal / divisor,
+  //       caution: metricConfig.caution / divisor,
+  //       alert: metricConfig.alert / divisor,
+  //     };
+  //   }
+
+  //   console.warn(
+  //     "Missing or unkown aggregation mode (sum/avg) for metric:",
+  //     metricConfig,
+  //   );
+  //   return null;
+  // }
+
+  // UPLOT PLUGIN // converts the legend into a simple tooltip
+  function legendAsTooltipPlugin({
+    className,
+    style = { backgroundColor: "rgba(255, 249, 196, 0.92)", color: "black" },
+  } = {}) {
+    let legendEl;
+    const dataSize = metricData.length;
+
+    function init(u, opts) {
+      legendEl = u.root.querySelector(".u-legend");
+
+      legendEl.classList.remove("u-inline");
+      className && legendEl.classList.add(className);
+
+      uPlot.assign(legendEl.style, {
+        minWidth: "100px",
+        textAlign: "left",
+        pointerEvents: "none",
+        display: "none",
+        position: "absolute",
+        left: 0,
+        top: 0,
+        zIndex: 100,
+        boxShadow: "2px 2px 10px rgba(0,0,0,0.5)",
+        ...style,
+      });
+
+      // conditional hide series color markers:
+      if (
+        // useStatsSeries || // Min/Max/Median Self-Explanatory
+        dataSize === 1 || // Only one Y-Dataseries
+        dataSize > 8 // More than 8 Y-Dataseries
+      ) {
+        const idents = legendEl.querySelectorAll(".u-marker");
+        for (let i = 0; i < idents.length; i++)
+          idents[i].style.display = "none";
+      }
+
+      const overEl = u.over;
+      overEl.style.overflow = "visible";
+
+      // move legend into plot bounds
+      overEl.appendChild(legendEl);
+
+      // show/hide tooltip on enter/exit
+      overEl.addEventListener("mouseenter", () => {
+        legendEl.style.display = null;
+      });
+      overEl.addEventListener("mouseleave", () => {
+        legendEl.style.display = "none";
+      });
+
+      // let tooltip exit plot
+      // overEl.style.overflow = "visible";
+    }
+
+    function update(u) {
+      const { left, top } = u.cursor;
+      const internalWidth = u?.over?.querySelector(".u-legend")?.offsetWidth ? u.over.querySelector(".u-legend").offsetWidth : 0;
+      if (enableFlip && (left < (width/2))) {
+        legendEl.style.transform = "translate(" + (left + 15) + "px, " + (top + 15) + "px)";
+      } else {
+        legendEl.style.transform = "translate(" + (left - internalWidth - 15) + "px, " + (top + 15) + "px)";
+      }
+    }
+
+    if (dataSize <= 12 ) { // || useStatsSeries) {
+      return {
+        hooks: {
+          init: init,
+          setCursor: update,
+        },
+      };
+    } else {
+      // Setting legend-opts show/live as object with false here will not work ...
+      return {};
+    }
+  }
+
+  // RETURN BG COLOR FROM THRESHOLD
+  // function backgroundColor() {
+  //   if (
+  //     clusterCockpitConfig.plotConfiguration_colorBackground == false ||
+  //     // !thresholds ||
+  //     !(series && series.every((s) => s.statistics != null))
+  //   )
+  //     return backgroundColors.normal;
+
+  //   let cond =
+  //     thresholds.alert < thresholds.caution
+  //       ? (a, b) => a <= b
+  //       : (a, b) => a >= b;
+
+  //   let avg =
+  //     series.reduce((sum, series) => sum + series.statistics.avg, 0) /
+  //     series.length;
+
+  //   if (Number.isNaN(avg)) return backgroundColors.normal;
+
+  //   if (cond(avg, thresholds.alert)) return backgroundColors.alert;
+
+  //   if (cond(avg, thresholds.caution)) return backgroundColors.caution;
+
+  //   return backgroundColors.normal;
+  // }
+
+  function lineColor(i, n) {
+    if (n && n >= lineColors.length) return lineColors[i % lineColors.length];
+    else return lineColors[Math.floor((i / n) * lineColors.length)];
+  }
+
+  function render(ren_width, ren_height) {
+    // Set Options
+    const opts = {
+      width,
+      height,
+      title: 'Cluster Utilization',
+      plugins: [legendAsTooltipPlugin()],
+      series: plotSeries,
+      axes: [
+        {
+          scale: "x",
+          space: 35,
+          incrs: timeIncrs(timestep, maxX, forNode),
+          label: "Time",
+          values: (_, vals) => vals.map((v) => formatDurationTime(v, forNode)),
+        },
+        {
+          scale: "y1",
+          grid: { show: true },
+          label: `${metricData[0]?.name} (${metricData[0]?.unit?.prefix}${metricData[0]?.unit?.base})`,
+          values: (u, vals) => vals.map((v) => formatNumber(v)),
+        },
+        {
+          side: 1,
+          scale: "y2",
+          grid: { show: false },
+          label: `${metricData[1]?.name} (${metricData[1]?.unit?.prefix}${metricData[1]?.unit?.base})`,
+          values: (u, vals) => vals.map((v) => formatNumber(v)),
+        },
+      ],
+      // bands: plotBands,
+      padding: [5, 10, -20, 0],
+      hooks: {
+        // init: [
+        //   (u) => {
+        //     /* IF Zoom Enabled */
+        //     if (resampleConfig && !forNode) {
+        //       u.over.addEventListener("dblclick", (e) => {
+        //         // console.log('Dispatch: Zoom Reset')
+        //         onZoom({
+        //           lastZoomState: {
+        //             x: { time: false },
+        //             y: { auto: true }
+        //           }
+        //         });
+        //       });
+        //     };
+        //   },
+        // ],
+        draw: [
+          (u) => {
+            // Draw plot type label:
+            let textl = `Cluster ${cluster}`
+            // let textl = `${scope}${plotSeries.length > 2 ? "s" : ""}${
+            //   useStatsSeries
+            //     ? (usesMeanStatsSeries ? ": min/mean/max" : ": min/median/max")
+            //     : metricConfig != null && scope != metricConfig.scope
+            //       ? ` (${metricConfig.aggregation})`
+            //       : ""
+            // }`;
+            let textr = `Sums of ${numNodes} nodes`
+            //let textr = `${isShared && scope != "core" && scope != "accelerator" ? "[Shared]" : ""}`;
+            u.ctx.save();
+            u.ctx.textAlign = "start"; // 'end'
+            u.ctx.fillStyle = "black";
+            u.ctx.fillText(textl, u.bbox.left + 10, u.bbox.top + (forNode ? 0 : 10));
+            u.ctx.textAlign = "end";
+            u.ctx.fillStyle = "black";
+            u.ctx.fillText(
+              textr,
+              u.bbox.left + u.bbox.width - 10,
+              u.bbox.top + (forNode ? 0 : 10),
+            );
+            // u.ctx.fillText(text, u.bbox.left + u.bbox.width - 10, u.bbox.top + u.bbox.height - 10) // Recipe for bottom right
+
+            // if (!thresholds) {
+              u.ctx.restore();
+              return;
+            // }
+
+            // let y = u.valToPos(thresholds.normal, "y", true);
+            // u.ctx.save();
+            // u.ctx.lineWidth = lineWidth;
+            // u.ctx.strokeStyle = normalLineColor;
+            // u.ctx.setLineDash([5, 5]);
+            // u.ctx.beginPath();
+            // u.ctx.moveTo(u.bbox.left, y);
+            // u.ctx.lineTo(u.bbox.left + u.bbox.width, y);
+            // u.ctx.stroke();
+            // u.ctx.restore();
+          },
+        ],
+        // setScale: [
+        //   (u, key) => { // If ZoomResample is Configured && Not System/Node View
+        //     if (resampleConfig && !forNode && key === 'x') {
+        //       const numX = (u.series[0].idxs[1] - u.series[0].idxs[0])
+        //       if (numX <= resampleTrigger && timestep !== resampleMinimum) {
+        //         /* Get closest zoom level; prevents multiple iterative zoom requests for big zoom-steps (e.g. 600 -> 300 -> 120 -> 60) */
+        //         // Which resolution to theoretically request to achieve 30 or more visible data points:
+        //         const target = (numX * timestep) / resampleTrigger
+        //         // Which configured resolution actually matches the closest to theoretical target:
+        //         const closest = resampleResolutions.reduce(function(prev, curr) {
+        //           return (Math.abs(curr - target) < Math.abs(prev - target) ? curr : prev);
+        //         });
+        //         // Prevents non-required dispatches
+        //         if (timestep !== closest) {
+        //           // console.log('Dispatch: Zoom with Res from / to', timestep, closest)
+        //           onZoom({
+        //             newRes: closest,
+        //             lastZoomState: u?.scales,
+        //             lastThreshold: thresholds?.normal
+        //           });
+        //         }
+        //       } else {
+        //         // console.log('Dispatch: Zoom Update States')
+        //         onZoom({
+        //           lastZoomState: u?.scales,
+        //           lastThreshold: thresholds?.normal
+        //         });
+        //       };
+        //     };
+        //   },
+        // ]
+      },
+      scales: {
+        x: { time: false },
+        y1: { auto: true },
+        y1: { auto: true },
+      },
+      legend: {
+        // Display legend until max 12 Y-dataseries
+        show: true, // metricData.length <= 12 || useStatsSeries,
+        live: true // But This Plot always for 2 Data-Series
+      },
+      cursor: { 
+        drag: { x: true, y: true },
+      }
+    };
+
+    // Handle Render
+    if (!uplot) {
+      opts.width = ren_width;
+      opts.height = ren_height;
+      
+      // if (plotSync) {
+      //   opts.cursor.sync = { 
+      //     key: plotSync.key,
+      //     scales: ["x", null],
+      //   }
+      // }
+
+      // if (zoomState && metricConfig?.aggregation == "avg") {
+      //   opts.scales = {...zoomState}
+      // } else if (zoomState && metricConfig?.aggregation == "sum") {
+      //   // Allow Zoom In === Ymin changed
+      //   if (zoomState.y.min !== 0) { // scope change?: only use zoomState if thresholds match
+      //     if ((thresholdState === thresholds?.normal)) { opts.scales = {...zoomState} };
+      //   } // else: reset scaling to default
+      // }
+
+      uplot = new uPlot(opts, plotData, plotWrapper);
+    } else {
+      uplot.setSize({ width: ren_width, height: ren_height });
+    }
+  }
+
+  function onSizeChange(chg_width, chg_height) {
+    if (!uplot) return;
+    if (timeoutId != null) clearTimeout(timeoutId);
+    timeoutId = setTimeout(() => {
+      timeoutId = null;
+      render(chg_width, chg_height);
+    }, renderSleepTime);
+  }
+
+  /* On Mount */
+  onMount(() => {
+    if (plotWrapper) {
+      render(width, height);
+    }
+  });
+
+  /* On Destroy */
+  onDestroy(() => {
+    if (timeoutId != null) clearTimeout(timeoutId);
+    if (uplot) uplot.destroy();
+  });
+
+</script>
+
+<!-- Define $width Wrapper and NoData Card -->
+{#if metricData[0]?.data && metricData[0]?.data?.length > 0}
+  <div bind:this={plotWrapper} bind:clientWidth={width}
+        class={forNode ? 'py-2 rounded' : 'rounded'}
+  ></div>
+{:else}
+  <Card body color="warning" class="mx-4"
+    >Cannot render plot: No series data returned for <code>{cluster}</code></Card
+  >
+{/if}

web/frontend/src/generic/plots/Pie.svelte

@@ -59,7 +59,15 @@
       'rgb(135,133,0)',
       'rgb(0,167,108)',
       'rgb(189,189,189)',
-    ]
+    ],
+    nodeStates: {
+      allocated: "rgba(0, 128, 0, 0.75)",
+      down: "rgba(255, 0, 0, 0.75)",
+      idle: "rgba(0, 0, 255, 0.75)",
+      reserved: "rgba(255, 0, 255, 0.75)",
+      mixed: "rgba(255, 215, 0, 0.75)",
+      unknown: "rgba(0, 0, 0, 0.75)"
+    }
   }
 </script>
 
@@ -77,6 +85,7 @@
     entities,
     displayLegend = false,
     useAltColors = false,
+    fixColors = null
   } = $props();
 
   /* Const Init */
@@ -98,6 +107,8 @@
       c = [...colors['colorblind']];
     } else if (useAltColors) {
       c = [...colors['alternative']];
+    } else if (fixColors?.length > 0) {
+      c = [...fixColors];
     } else {
       c = [...colors['default']];
     }

web/frontend/src/generic/plots/Roofline.svelte

@@ -34,14 +34,18 @@
     nodesData = null,
     cluster = null,
     subCluster = null,
+    fixTitle = null,
+    yMinimum = null,
     allowSizeChange = false,
     useColors = true,
+    useLegend = true,
+    colorBackground = false,
     width = 600,
     height = 380,
   } = $props();
 
   /* Const Init */
-  const lineWidth = clusterCockpitConfig.plotConfiguration_lineWidth;
+  const lineWidth = 2 // clusterCockpitConfig.plotConfiguration_lineWidth;
   const cbmode = clusterCockpitConfig?.plotConfiguration_colorblindMode || false;
 
   /* Var Init */
@@ -293,7 +297,7 @@
           } else {
             // No Colors: Use Black
             u.ctx.strokeStyle = "rgb(0, 0, 0)";
-            u.ctx.fillStyle = "rgba(0, 0, 0, 0.5)";
+            u.ctx.fillStyle = colorBackground ? "rgb(0, 0, 0)" : "rgba(0, 0, 0, 0.5)";
           }
 
           // Get Values
@@ -526,6 +530,7 @@
     let plotTitle = "CPU Roofline Diagram";
     if (jobsData) plotTitle = "Job Average Roofline Diagram";
     if (nodesData) plotTitle = "Node Average Roofline Diagram";
+    if (fixTitle) plotTitle = fixTitle
 
     if (roofData) {
       const opts = {
@@ -534,7 +539,7 @@
         width: width,
         height: height,
         legend: {
-          show: true,
+          show: useLegend,
         },
         cursor: { 
           dataIdx: (u, seriesIdx) => {
@@ -616,7 +621,7 @@
           },
           y: {
             range: [
-              0.01,
+              yMinimum ? yMinimum : 0.01,
               subCluster?.flopRateSimd?.value
                 ? nearestThousand(subCluster.flopRateSimd.value)
                 : 10000,
@@ -646,7 +651,7 @@
         hooks: {
           // setSeries: [ (u, seriesIdx) => console.log('setSeries', seriesIdx) ],
           // setLegend: [ u => console.log('setLegend', u.legend.idxs) ],
-          drawClear: [
+          drawClear: [ // drawClear hook which fires before anything exists, so will render under the grid
             (u) => {
               qt = qt || new Quadtree(0, 0, u.bbox.width, u.bbox.height);
               qt.clear();
@@ -658,7 +663,7 @@
               });
             },
           ],
-          draw: [
+          drawAxes: [ // drawAxes hook, which fires after axes and grid have been rendered
             (u) => {
               // draw roofs when subCluster set
               if (subCluster != null) {
@@ -668,6 +673,7 @@
                 u.ctx.lineWidth = lineWidth;
                 u.ctx.beginPath();
 
+                // Get Values
                 const ycut = 0.01 * subCluster.memoryBandwidth.value;
                 const scalarKnee =
                   (subCluster.flopRateScalar.value - ycut) /
@@ -675,19 +681,20 @@
                 const simdKnee =
                   (subCluster.flopRateSimd.value - ycut) /
                   subCluster.memoryBandwidth.value;
-                const scalarKneeX = u.valToPos(scalarKnee, "x", true), // Value, axis, toCanvasPixels
-                  simdKneeX = u.valToPos(simdKnee, "x", true),
-                  flopRateScalarY = u.valToPos(
-                    subCluster.flopRateScalar.value,
-                    "y",
-                    true,
-                  ),
-                  flopRateSimdY = u.valToPos(
-                    subCluster.flopRateSimd.value,
-                    "y",
-                    true,
-                  );
 
+                // Get Const Coords
+                const originX = u.valToPos(0.01, "x", true);
+                const originY = u.valToPos(yMinimum ? yMinimum : 0.01, "y", true);
+
+                const outerX = u.valToPos(1000, "x", true); // rightmost x in plot coords
+
+                const scalarKneeX = u.valToPos(scalarKnee, "x", true) // Value, axis, toCanvasPixels
+                const simdKneeX = u.valToPos(simdKnee, "x", true)
+                
+                const flopRateScalarY = u.valToPos(subCluster.flopRateScalar.value, "y", true)
+                const flopRateSimdY = u.valToPos(subCluster.flopRateSimd.value, "y", true);
+
+                /* Render Lines */
                 if (
                   scalarKneeX <
                   width * window.devicePixelRatio -
@@ -727,10 +734,10 @@
                   y1,
                   x2,
                   y2,
-                  u.valToPos(0.01, "x", true),
+                  originX, // x3; X-Axis Start Coord-X
-                  u.valToPos(1.0, "y", true), // X-Axis Start Coords
+                  originY, // y3; X-Axis Start Coord-Y
-                  u.valToPos(1000, "x", true),
+                  outerX,  // x4; X-Axis End Coord-X
-                  u.valToPos(1.0, "y", true), // X-Axis End Coords
+                  originY, // y4; X-Axis End Coord-Y
                 );
 
                 if (xAxisIntersect.x > x1) {
@@ -745,6 +752,144 @@
                 u.ctx.stroke();
                 // Reset grid lineWidth
                 u.ctx.lineWidth = 0.15;
+
+                /* Render Area */
+                if (colorBackground) {
+
+                  u.ctx.beginPath();
+                  // Additional Coords for Colored Regions
+                  const yhalf = u.valToPos(ycut/2, "y", true)
+                  const simdShift = u.valToPos(simdKnee*1.75, "x", true)
+
+                  let upperBorderIntersect = lineIntersect(
+                    x1,
+                    y1,
+                    x2,
+                    y2,
+                    originX, // x3; X-Axis Start Coord-X
+                    flopRateSimdY*1.667, // y3; X-Axis Start Coord-Y
+                    outerX,  // x4; X-Axis End Coord-X
+                    flopRateSimdY*1.667, // y4; X-Axis End Coord-Y
+                  );
+
+                  let lowerBorderIntersect = lineIntersect(
+                    x1,
+                    y1,
+                    x2,
+                    y2,
+                    originX, // x3; X-Axis Start Coord-X
+                    flopRateScalarY*1.1667, // y3; X-Axis Start Coord-Y
+                    outerX,  // x4; X-Axis End Coord-X
+                    flopRateScalarY*1.1667, // y4; X-Axis End Coord-Y
+                  );
+
+                  let helperUpperBorderIntersect = lineIntersect(
+                    x1,
+                    yhalf,
+                    simdShift,
+                    y2,
+                    originX, // x3; X-Axis Start Coord-X
+                    flopRateSimdY*1.667, // y3; X-Axis Start Coord-Y
+                    outerX,  // x4; X-Axis End Coord-X
+                    flopRateSimdY*1.667, // y4; X-Axis End Coord-Y
+                  );
+
+                  let helperLowerBorderIntersect = lineIntersect(
+                    x1,
+                    yhalf,
+                    simdShift,
+                    y2,
+                    originX, // x3; X-Axis Start Coord-X
+                    flopRateScalarY*1.1667, // y3; X-Axis Start Coord-Y
+                    outerX,  // x4; X-Axis End Coord-X
+                    flopRateScalarY*1.1667, // y4; X-Axis End Coord-Y
+                  );
+
+                  let helperLowerBorderIntersectTop = lineIntersect(
+                    x1,
+                    yhalf,
+                    simdShift,
+                    y2,
+                    scalarKneeX, // x3; X-Axis Start Coord-X
+                    flopRateScalarY, // y3; X-Axis Start Coord-Y
+                    outerX,  // x4; X-Axis End Coord-X
+                    flopRateScalarY, // y4; X-Axis End Coord-Y
+                  );
+
+                  // Diagonal Helper
+                  u.ctx.moveTo(x1, yhalf);
+                  u.ctx.lineTo(simdShift, y2);
+                  // Upper Simd Helper
+                  u.ctx.moveTo(upperBorderIntersect.x, flopRateSimdY*1.667);
+                  u.ctx.lineTo(outerX, flopRateSimdY*1.667);
+                  // Lower Scalar Helper
+                  u.ctx.moveTo(lowerBorderIntersect.x, flopRateScalarY*1.1667);
+                  u.ctx.lineTo(outerX, flopRateScalarY*1.1667);
+
+                  u.ctx.stroke();
+
+                  /* Color Regions */
+                  // MemoryBound
+                  u.ctx.save();
+                  u.ctx.beginPath();
+                  u.ctx.lineTo(x1, y1);        // YCut
+                  u.ctx.lineTo(x2, y2);        // Upper Knee
+                  u.ctx.lineTo(simdShift, y2); // Upper Helper Knee
+                  u.ctx.lineTo(x1, yhalf);     // Half yCut
+                  u.ctx.closePath();
+                  u.ctx.fillStyle = "rgba(255, 200, 0, 0.4)"; // Yellow
+                  u.ctx.fill();
+                  u.ctx.restore();
+
+                  // Compute Lower
+                  u.ctx.save();
+                  u.ctx.beginPath();
+                  u.ctx.moveTo(lowerBorderIntersect.x, flopRateScalarY*1.1667); // Lower Helper Knee
+                  u.ctx.lineTo(scalarKneeX, flopRateScalarY);                   // Lower Knee
+                  u.ctx.lineTo(outerX, flopRateScalarY);                        // Outer Border
+                  u.ctx.lineTo(outerX, flopRateScalarY*1.1667);                 // Outer Lower Helper Border
+                  u.ctx.closePath();
+                  u.ctx.fillStyle = "rgba(0, 180, 255, 0.4)"; // Cyan Blue
+                  u.ctx.fill();
+                  u.ctx.restore();
+
+                  // Compute Upper
+                  u.ctx.save();
+                  u.ctx.beginPath();
+                  u.ctx.moveTo(upperBorderIntersect.x, flopRateSimdY*1.667); // Upper Helper Knee
+                  u.ctx.lineTo(simdKneeX, flopRateSimdY);                    // Upper Knee
+                  u.ctx.lineTo(outerX, flopRateSimdY);                       // Outer Border
+                  u.ctx.lineTo(outerX, flopRateSimdY*1.667);                 // Outer Upper Helper Border
+                  u.ctx.closePath();
+                  u.ctx.fillStyle = "rgba(0, 180, 255, 0.4)"; // Cyan Blue
+                  u.ctx.fill();
+                  u.ctx.restore();
+
+                  // Nomansland Lower
+                  u.ctx.save();
+                  u.ctx.beginPath();
+                  u.ctx.moveTo(originX, originY);                                     // Origin
+                  u.ctx.lineTo(originX, yhalf);                                       // YCut Half
+                  u.ctx.lineTo(helperLowerBorderIntersect.x, flopRateScalarY*1.1667); // Lower Inner Helper Knee
+                  u.ctx.lineTo(outerX, flopRateScalarY*1.1667);                       // Lower Inner Border
+                  u.ctx.lineTo(outerX, originY);                                      // Lower Right Corner
+                  u.ctx.closePath();
+                  u.ctx.fillStyle = "rgba(255, 50, 50, 0.1)"; // Red Light
+                  u.ctx.fill();
+                  u.ctx.restore();
+
+                  // Nomansland Upper
+                  u.ctx.save();
+                  u.ctx.beginPath();
+                  u.ctx.moveTo(helperLowerBorderIntersectTop.x, flopRateScalarY);  // Lower Knee Top
+                  u.ctx.lineTo(helperUpperBorderIntersect.x, flopRateSimdY*1.667); // Upper Helper Knee
+                  u.ctx.lineTo(outerX, flopRateSimdY*1.667);                       // Upper Inner Border
+                  u.ctx.lineTo(outerX, flopRateScalarY);                           // Lower Knee Border
+                  u.ctx.closePath();
+                  u.ctx.fillStyle = "rgba(255, 50, 50, 0.1)"; // Red Light
+                  u.ctx.fill();
+                  u.ctx.restore();
+                }
               }
 
               /* Render Scales */
@@ -852,5 +997,5 @@
 {#if roofData != null}
   <div bind:this={plotWrapper} class="p-2"></div>
 {:else}
-  <Card class="mx-4" body color="warning">Cannot render roofline: No data!</Card>
+  <Card class="mx-4 my-2" body color="warning">Cannot render roofline: No data!</Card>
 {/if}

web/frontend/src/generic/plots/RooflineLegacy.svelte

@@ -315,10 +315,10 @@
                   y1,
                   x2,
                   y2,
-                  u.valToPos(0.01, "x", true),
+                  u.valToPos(0.01, "x", true), // x3; X-Axis Start Coord-X
-                  u.valToPos(1.0, "y", true), // X-Axis Start Coords
+                  u.valToPos(1.0, "y", true), // y3; X-Axis Start Coord-Y
-                  u.valToPos(1000, "x", true),
+                  u.valToPos(1000, "x", true), // x4; X-Axis End Coord-X
-                  u.valToPos(1.0, "y", true), // X-Axis End Coords
+                  u.valToPos(1.0, "y", true), // y4; X-Axis End Coord-Y
                 );
 
                 if (xAxisIntersect.x > x1) {

web/frontend/src/generic/plots/Stacked.svelte

@@ -39,72 +39,72 @@
       label: "Full",
       scale: "y",
       width: lineWidth,
-      fill: cbmode ? "rgba(0, 110, 0, 0.4)" : "rgba(0, 128, 0, 0.4)",
+      fill: cbmode ? "rgba(0, 110, 0, 0.6)" : "rgba(0, 128, 0, 0.6)",
       stroke: cbmode ? "rgb(0, 110, 0)" : "green",
     },
     partial: {
       label: "Partial",
       scale: "y",
       width: lineWidth,
-      fill: cbmode ? "rgba(235, 172, 35, 0.4)" : "rgba(255, 215, 0, 0.4)",
+      fill: cbmode ? "rgba(235, 172, 35, 0.6)" : "rgba(255, 215, 0, 0.6)",
       stroke: cbmode ? "rgb(235, 172, 35)" : "gold",
     },
     failed: {
       label: "Failed",
       scale: "y",
       width: lineWidth,
-      fill: cbmode ? "rgb(181, 29, 20, 0.4)" : "rgba(255, 0, 0, 0.4)",
+      fill: cbmode ? "rgb(181, 29, 20, 0.6)" : "rgba(255, 0, 0, 0.6)",
       stroke: cbmode ? "rgb(181, 29, 20)" : "red",
     },
     idle: {
       label: "Idle",
       scale: "y",
       width: lineWidth,
-      fill: cbmode ? "rgba(0, 140, 249, 0.4)" : "rgba(0, 0, 255, 0.4)",
+      fill: cbmode ? "rgba(0, 140, 249, 0.6)" : "rgba(0, 0, 255, 0.6)",
       stroke: cbmode ? "rgb(0, 140, 249)" : "blue",
     },
     allocated: {
       label: "Allocated",
       scale: "y",
       width: lineWidth,
-      fill: cbmode ? "rgba(0, 110, 0, 0.4)" : "rgba(0, 128, 0, 0.4)",
+      fill: cbmode ? "rgba(0, 110, 0, 0.6)" : "rgba(0, 128, 0, 0.6)",
       stroke: cbmode ? "rgb(0, 110, 0)" : "green",
     },
     reserved: {
       label: "Reserved",
       scale: "y",
       width: lineWidth,
-      fill: cbmode ? "rgba(209, 99, 230, 0.4)" : "rgba(255, 0, 255, 0.4)",
+      fill: cbmode ? "rgba(209, 99, 230, 0.6)" : "rgba(255, 0, 255, 0.6)",
       stroke: cbmode ? "rgb(209, 99, 230)" : "magenta",
     },
     mixed: {
       label: "Mixed",
       scale: "y",
       width: lineWidth,
-      fill: cbmode ? "rgba(235, 172, 35, 0.4)" : "rgba(255, 215, 0, 0.4)",
+      fill: cbmode ? "rgba(235, 172, 35, 0.6)" : "rgba(255, 215, 0, 0.6)",
       stroke: cbmode ? "rgb(235, 172, 35)" : "gold",
     },
     down: {
       label: "Down",
       scale: "y",
       width: lineWidth,
-      fill: cbmode ? "rgba(181, 29 ,20, 0.4)" : "rgba(255, 0, 0, 0.4)",
+      fill: cbmode ? "rgba(181, 29 ,20, 0.6)" : "rgba(255, 0, 0, 0.6)",
       stroke: cbmode ? "rgb(181, 29, 20)" : "red",
     },
     unknown: {
       label: "Unknown",
       scale: "y",
       width: lineWidth,
-      fill: "rgba(0, 0, 0, 0.4)",
+      fill: "rgba(0, 0, 0, 0.6)",
       stroke: "black",
     }
   };
 
   // Data Prep For uPlot
-  const sortedData = data.sort((a, b) => a.state.localeCompare(b.state));
+  const sortedData = data?.sort((a, b) => a.state.localeCompare(b.state)) || [];
   const collectLabel = sortedData.map(d => d.state);
   // Align Data to Timesteps, Introduces 'undefied' as placeholder, reiterate and set those to 0
-  const collectData  = uPlot.join(sortedData.map(d => [d.times, d.counts])).map(d => d.map(i => i ? i : 0));
+  const collectData = (sortedData.length > 0) ? uPlot.join(sortedData.map(d => [d.times, d.counts])).map(d => d.map(i => i ? i : 0)) : [];
 
   // STACKED CHART FUNCTIONS //
   function stack(data, omit) {
@@ -156,7 +156,7 @@
         {
           scale: "y",
           grid: { show: true },
-          labelFont: "sans-serif",
+          // labelFont: "sans-serif",
           label: ylabel + (yunit ? ` (${yunit})` : ''),
           // values: (u, vals) => vals.map((v) => formatNumber(v)),
         },
@@ -344,7 +344,7 @@
 </script>
 
 <!-- Define $width Wrapper and NoData Card -->
-{#if data && collectData[0].length > 0}
+{#if data && collectData.length > 0}
   <div bind:this={plotWrapper} bind:clientWidth={width}
         style="background-color: rgba(255, 255, 255, 1.0);" class="rounded"
   ></div>

web/frontend/src/header/NavbarLinks.svelte

@@ -64,6 +64,34 @@
           {/each}
         </DropdownMenu>
       </Dropdown>
+    {:else if item.title === 'Status'}
+      <Dropdown nav inNavbar {direction}>
+        <DropdownToggle nav caret>
+          <Icon name={item.icon} />
+          {item.title}
+        </DropdownToggle>
+        <DropdownMenu class="dropdown-menu-lg-end">
+          {#each clusters as cluster}
+            <Dropdown nav direction="right">
+              <DropdownToggle nav caret class="dropdown-item py-1 px-2">
+                {cluster.name}
+              </DropdownToggle>
+              <DropdownMenu>
+                <DropdownItem class="py-1 px-2"
+                  href={item.href + cluster.name}
+                >
+                  Status Dashboard
+                </DropdownItem>
+                <DropdownItem class="py-1 px-2"
+                  href={item.href + 'detail/' + cluster.name}
+                >
+                  Status Details
+                </DropdownItem>
+              </DropdownMenu>
+            </Dropdown>
+          {/each}
+        </DropdownMenu>
+      </Dropdown>
     {:else}
       <Dropdown nav inNavbar {direction}>
         <DropdownToggle nav caret>

web/frontend/src/status.entrypoint.js

@@ -6,6 +6,7 @@ mount(Status, {
     target: document.getElementById('svelte-app'),
     props: {
         presetCluster: infos.cluster,
+        displayType: displayType,
     },
     context: new Map([
             ['cc-config', clusterCockpitConfig]

web/frontend/src/status/DashDetails.svelte

@@ -0,0 +1,82 @@
+<!--
+  @component Main cluster status view component; renders current system-usage information
+
+  Properties:
+  - `presetCluster String`: The cluster to show status information for
+-->
+
+ <script>
+  import {
+    getContext
+  } from "svelte"
+  import {
+    init,
+  } from "../generic/utils.js";
+  import {
+    Row,
+    Col,
+    Card,
+    CardBody,
+    TabContent,
+    TabPane,
+    Spinner
+  } from "@sveltestrap/sveltestrap";
+
+  import StatusDash from "./dashdetails/StatusDash.svelte";
+  import UsageDash from "./dashdetails/UsageDash.svelte";
+  import StatisticsDash from "./dashdetails/StatisticsDash.svelte";
+
+  /* Svelte 5 Props */
+  let {
+    presetCluster,
+  } = $props();
+
+  /*Const Init */
+  const { query: initq } = init();
+  const useCbColors = getContext("cc-config")?.plotConfiguration_colorblindMode || false
+</script>
+
+<!-- Loading indicator & Refresh -->
+
+<Row cols={1} class="mb-2">
+  <Col>
+    <h3 class="mb-0">Current Status of Cluster "{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}"</h3>
+  </Col>
+</Row>
+
+
+{#if $initq.fetching}
+  <Row cols={1} class="text-center mt-3">
+    <Col>
+      <Spinner />
+    </Col>
+  </Row>
+{:else if $initq.error}
+  <Row cols={1} class="text-center mt-3">
+    <Col>  
+      <Card body color="danger">{$initq.error.message}</Card>
+    </Col>
+  </Row>
+{:else}
+  <Card class="overflow-auto" style="height: auto;">
+    <TabContent>
+      <TabPane tabId="status-dash" tab="Status" active>
+        <CardBody>
+          <StatusDash clusters={$initq.data.clusters} {presetCluster} {useCbColors} useAltColors></StatusDash>
+        </CardBody>
+      </TabPane>
+
+      <TabPane tabId="usage-dash" tab="Usage">
+        <CardBody>
+          <UsageDash {presetCluster} {useCbColors}></UsageDash>
+        </CardBody>
+      </TabPane>
+      
+      <TabPane tabId="metric-dash" tab="Statistics">
+        <CardBody>
+          <StatisticsDash {presetCluster} {useCbColors}></StatisticsDash>
+        </CardBody>
+      </TabPane>
+    </TabContent>
+  </Card>
+{/if}

web/frontend/src/status/DashInternal.svelte

@@ -0,0 +1,600 @@
+<!--
+  @component Main cluster status view component; renders current system-usage information
+
+  Properties:
+  - `presetCluster String`: The cluster to show status information for
+-->
+
+ <script>
+  import {
+    getContext
+  } from "svelte"
+  import {
+    queryStore,
+    gql,
+    getContextClient,
+  } from "@urql/svelte";
+  import {
+    init,
+    scramble,
+    scrambleNames,
+  } from "../generic/utils.js";
+  import {
+    formatDurationTime,
+    formatNumber,
+  } from "../generic/units.js";
+  import {
+    Row,
+    Col,
+    Card,
+    CardTitle,
+    CardHeader,
+    CardBody,
+    Spinner,
+    Table,
+    Progress,
+    Icon,
+  } from "@sveltestrap/sveltestrap";
+  import Roofline from "../generic/plots/Roofline.svelte";
+  import Pie, { colors } from "../generic/plots/Pie.svelte";
+  import Stacked from "../generic/plots/Stacked.svelte";
+  import DoubleMetric from "../generic/plots/DoubleMetricPlot.svelte";
+  import Refresher from "../generic/helper/Refresher.svelte";
+
+  /* Svelte 5 Props */
+  let {
+    presetCluster,
+  } = $props();
+
+  /*Const Init */
+  const { query: initq } = init();
+  const client = getContextClient();
+  const useCbColors = getContext("cc-config")?.plotConfiguration_colorblindMode || false
+
+  /* States */
+  let pagingState = $state({page: 1, itemsPerPage: 10}) // Top 10
+  let from = $state(new Date(Date.now() - 5 * 60 * 1000));
+  let clusterFrom = $state(new Date(Date.now() - (8 * 60 * 60 * 1000)));
+  let to = $state(new Date(Date.now()));
+  let stackedFrom = $state(Math.floor(Date.now() / 1000) - 14400);
+  let colWidthJobs = $state(0);
+  let colWidthRoof = $state(0);
+  let colWidthTotals =$state(0);
+  let colWidthStacked1 = $state(0);
+  let colWidthStacked2 = $state(0);
+
+  /* Derived */
+  // States for Stacked charts
+  const statesTimed = $derived(queryStore({
+    client: client,
+    query: gql`
+      query ($filter: [NodeFilter!], $typeNode: String!, $typeHealth: String!) {
+        nodeStates: nodeStatesTimed(filter: $filter, type: $typeNode) {
+          state
+          counts
+          times
+        }
+        healthStates: nodeStatesTimed(filter: $filter, type: $typeHealth) {
+          state
+          counts
+          times
+        }
+      }
+    `,
+    variables: {
+      filter: { cluster: { eq: presetCluster }, timeStart: stackedFrom},
+      typeNode: "node",
+      typeHealth: "health"
+    },
+    requestPolicy: "network-only"
+  }));
+
+  // Note: nodeMetrics are requested on configured $timestep resolution
+  // Result: The latest 5 minutes (datapoints) for each node independent of job
+  const statusQuery = $derived(queryStore({
+    client: client,
+    query: gql`
+      query (
+        $cluster: String!
+        $metrics: [String!]
+        $from: Time!
+        $to: Time!
+        $clusterFrom: Time!
+        $jobFilter: [JobFilter!]!
+        $paging: PageRequest!
+        $sorting: OrderByInput!
+      ) {
+        # Node 5 Minute Averages for Roofline
+        nodeMetrics(
+          cluster: $cluster
+          metrics: $metrics
+          from: $from
+          to: $to
+        ) {
+          host
+          subCluster
+          metrics {
+            name
+            metric {
+              series {
+                statistics {
+                  avg
+                }
+              }
+            }
+          }
+        }
+        # Running Job Metric Average for Rooflines
+        jobsMetricStats(filter: $jobFilter, metrics: $metrics) {
+          id
+          jobId
+          duration
+          numNodes
+          numAccelerators
+          subCluster
+          stats {
+            name
+            data {
+              avg
+            }
+          }
+        }
+        # Get Jobs for Per-Node Counts
+        jobs(filter: $jobFilter, order: $sorting, page: $paging) {
+          items {
+            jobId
+            resources {
+              hostname
+            }
+          }
+          count
+        }
+        # Only counts shared nodes once 
+        allocatedNodes(cluster: $cluster) {
+          name
+          count
+        }
+        # totalNodes includes multiples if shared jobs: Info-Card Data
+        jobsStatistics(
+          filter: $jobFilter
+          page: $paging
+          sortBy: TOTALJOBS
+          groupBy: SUBCLUSTER
+        ) {
+          id
+          totalJobs
+          totalUsers
+          totalCores
+          totalAccs
+        }
+        # ClusterMetrics for doubleMetricPlot
+        clusterMetrics(
+          cluster: $cluster
+          metrics: $metrics
+          from: $clusterFrom
+          to: $to
+        ) {
+          nodeCount
+          metrics {
+            name
+            unit {
+              prefix
+              base
+            }
+            timestep
+            data
+          }
+        }
+      }
+    `,
+    variables: {
+      cluster: presetCluster,
+      metrics: ["flops_any", "mem_bw"], // Fixed names for roofline and status bars
+      from: from.toISOString(),
+      to: to.toISOString(),
+      clusterFrom: clusterFrom.toISOString(),
+      jobFilter: [{ state: ["running"] }, { cluster: { eq: presetCluster } }],
+      paging: { itemsPerPage: -1, page: 1 }, // Get all: -1
+      sorting: { field: "startTime", type: "col", order: "DESC" }
+    },
+    requestPolicy: "network-only"
+  }));
+
+  const topJobsQuery = $derived(queryStore({
+    client: client,
+    query: gql`
+      query (
+        $filter: [JobFilter!]!
+        $paging: PageRequest!
+      ) {
+        jobsStatistics(
+          filter: $filter
+          page: $paging
+          sortBy: TOTALJOBS
+          groupBy: PROJECT
+        ) {
+          id
+          totalJobs
+        }
+      }
+    `,
+    variables: {
+      filter: [{ state: ["running"] }, { cluster: { eq: presetCluster} }],
+      paging: pagingState // Top 10
+    },
+    requestPolicy: "network-only"
+  }));
+
+  const clusterInfo = $derived.by(() => {
+    if ($initq?.data?.clusters) {
+      let rawInfos = {};
+      let subClusters = $initq?.data?.clusters?.find((c) => c.name == presetCluster)?.subClusters || [];
+      for (let subCluster of subClusters) {
+        // Allocations
+        if (!rawInfos['allocatedNodes']) rawInfos['allocatedNodes'] = $statusQuery?.data?.allocatedNodes?.find(({ name }) => name == subCluster.name)?.count || 0;
+        else rawInfos['allocatedNodes'] += $statusQuery?.data?.allocatedNodes?.find(({ name }) => name == subCluster.name)?.count || 0;
+
+        if (!rawInfos['allocatedCores']) rawInfos['allocatedCores'] = $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalCores || 0;
+        else rawInfos['allocatedCores'] += $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalCores || 0;
+
+        if (!rawInfos['allocatedAccs']) rawInfos['allocatedAccs'] = $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalAccs || 0;
+        else rawInfos['allocatedAccs'] += $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalAccs || 0;
+
+        // Infos
+        if (!rawInfos['processorTypes']) rawInfos['processorTypes'] = subCluster?.processorType ? new Set([subCluster.processorType]) : new Set([]);
+        else rawInfos['processorTypes'].add(subCluster.processorType);
+
+        if (!rawInfos['activeUsers']) rawInfos['activeUsers'] = $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalUsers || 0;
+        else rawInfos['activeUsers'] += $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalUsers || 0;
+
+        if (!rawInfos['runningJobs']) rawInfos['runningJobs'] = $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalJobs || 0;
+        else rawInfos['runningJobs'] += $statusQuery?.data?.jobsStatistics?.find(({ id }) => id == subCluster.name)?.totalJobs || 0;
+
+        if (!rawInfos['totalNodes']) rawInfos['totalNodes'] = subCluster?.numberOfNodes || 0;
+        else rawInfos['totalNodes'] += subCluster?.numberOfNodes || 0;
+
+        if (!rawInfos['totalCores']) rawInfos['totalCores'] = (subCluster?.socketsPerNode * subCluster?.coresPerSocket * subCluster?.numberOfNodes) || 0;
+        else rawInfos['totalCores'] += (subCluster?.socketsPerNode * subCluster?.coresPerSocket * subCluster?.numberOfNodes) || 0;
+
+        if (!rawInfos['totalAccs']) rawInfos['totalAccs'] = (subCluster?.numberOfNodes * subCluster?.topology?.accelerators?.length) || 0;
+        else rawInfos['totalAccs'] += (subCluster?.numberOfNodes * subCluster?.topology?.accelerators?.length) || 0;
+          
+        // Units (Set Once)
+        if (!rawInfos['flopRateUnit']) rawInfos['flopRateUnit'] = subCluster.flopRateSimd.unit.prefix + subCluster.flopRateSimd.unit.base
+        if (!rawInfos['memBwRateUnit']) rawInfos['memBwRateUnit'] = subCluster.memoryBandwidth.unit.prefix + subCluster.memoryBandwidth.unit.base
+
+        // Get Maxima For Roofline Knee Render
+        if (!rawInfos['roofData']) {
+          rawInfos['roofData'] = {
+            flopRateScalar: {value: subCluster.flopRateScalar.value},
+            flopRateSimd: {value: subCluster.flopRateSimd.value},
+            memoryBandwidth: {value: subCluster.memoryBandwidth.value}
+          };
+        } else { 
+          rawInfos['roofData']['flopRateScalar']['value'] = Math.max(rawInfos['roofData']['flopRateScalar']['value'], subCluster.flopRateScalar.value)
+          rawInfos['roofData']['flopRateSimd']['value'] = Math.max(rawInfos['roofData']['flopRateSimd']['value'], subCluster.flopRateSimd.value)
+          rawInfos['roofData']['memoryBandwidth']['value'] = Math.max(rawInfos['roofData']['memoryBandwidth']['value'], subCluster.memoryBandwidth.value)
+        }
+      }
+
+      // Keymetrics (Data on Cluster-Scope)
+      let rawFlops = $statusQuery?.data?.nodeMetrics?.reduce((sum, node) =>
+        sum + (node.metrics.find((m) => m.name == 'flops_any')?.metric?.series[0]?.statistics?.avg || 0),
+        0, // Initial Value
+      ) || 0;
+      rawInfos['flopRate'] = Math.floor((rawFlops * 100) / 100)
+
+      let rawMemBw = $statusQuery?.data?.nodeMetrics?.reduce((sum, node) =>
+        sum + (node.metrics.find((m) => m.name == 'mem_bw')?.metric?.series[0]?.statistics?.avg || 0),
+        0, // Initial Value
+      ) || 0;
+      rawInfos['memBwRate'] = Math.floor((rawMemBw * 100) / 100)
+
+      return rawInfos
+    } else {
+      return {};
+    }
+  });
+
+  /* Functions */
+  function legendColors(targetIdx) {
+    // Reuses first color if targetIdx overflows
+    let c;
+      if (useCbColors) {
+        c = [...colors['colorblind']];
+      // } else if (useAltColors) {
+      //   c = [...colors['alternative']];
+      } else {
+        c = [...colors['default']];
+      }
+    return  c[(c.length + targetIdx) % c.length];
+  }
+
+  function transformJobsStatsToData(clusterData) {
+    /* c will contain values from 0 to 1 representing the duration */
+    let data = null
+    const x = [], y = [], c = [], day = 86400.0
+
+    if (clusterData) {
+      for (let i = 0; i < clusterData.length; i++) {
+        const flopsData = clusterData[i].stats.find((s) => s.name == "flops_any")
+        const memBwData = clusterData[i].stats.find((s) => s.name == "mem_bw")
+            
+        const f = flopsData.data.avg
+        const m = memBwData.data.avg
+        const d = clusterData[i].duration / day
+
+        const intensity = f / m
+        if (Number.isNaN(intensity) || !Number.isFinite(intensity))
+            continue
+
+        x.push(intensity)
+        y.push(f)
+        // Long Jobs > 1 Day: Use max Color
+        if (d > 1.0) c.push(1.0)
+        else c.push(d)
+      }
+    } else {
+        console.warn("transformJobsStatsToData: metrics for 'mem_bw' and/or 'flops_any' missing!")
+    }
+
+    if (x.length > 0 && y.length > 0 && c.length > 0) {
+        data = [null, [x, y], c] // for dataformat see roofline.svelte
+    }
+    return data
+  }
+
+  function transformJobsStatsToInfo(clusterData) {
+    if (clusterData) {
+        return clusterData.map((sc) => { return {id: sc.id, jobId: sc.jobId, numNodes: sc.numNodes, numAcc: sc?.numAccelerators? sc.numAccelerators : 0, duration: formatDurationTime(sc.duration)} })
+    } else {
+        console.warn("transformJobsStatsToInfo: jobInfo missing!")
+        return []
+    }
+  }
+
+</script>
+
+<Card style="height: 88vh;">
+  <CardBody class="align-content-center">
+    <Row>
+      <Col>
+        <Refresher
+          hideSelector
+          initially={60}
+          onRefresh={(interval) => {
+            from = new Date(Date.now() - 5 * 60 * 1000);
+            to = new Date(Date.now());
+            clusterFrom = new Date(Date.now() - (8 * 60 * 60 * 1000))
+            pagingState = { page:1, itemsPerPage: 10 };
+
+            if (interval) stackedFrom += Math.floor(interval / 1000);
+            else stackedFrom += 1 // Workaround: TimeSelection not linked, just trigger new data on manual refresh
+          }}
+        />
+      </Col>
+    </Row>
+    {#if $statusQuery.fetching || $statesTimed.fetching || $topJobsQuery.fetching}
+      <Row class="justify-content-center">
+        <Col xs="auto">
+          <Spinner />
+        </Col>
+      </Row>
+
+    {:else if $statusQuery.error || $statesTimed.error || $topJobsQuery.error}
+      <Row>
+        {#if $statusQuery.error}
+          <Col>
+            <Card color="danger"><CardBody>Error Requesting Status Data: {$statusQuery.error.message}</CardBody></Card>
+          </Col>
+        {/if}
+        {#if $statesTimed.error}
+          <Col>
+            <Card color="danger"><CardBody>Error Requesting Node Scheduler States: {$statesTimed.error.message}</CardBody></Card>
+          </Col>
+        {/if}
+        {#if $topJobsQuery.error}
+          <Col>
+            <Card color="danger"><CardBody>Error Requesting Jobs By Project: {$topJobsQuery.error.message}</CardBody></Card>
+          </Col>
+        {/if}
+      </Row>
+
+    {:else}
+      <Row cols={{xs:1, md:2, xl: 3}}>
+        <Col> <!-- Info Card -->
+          <Card class="h-auto mt-1">
+            <CardHeader>
+              <CardTitle class="mb-0">Cluster "{presetCluster.charAt(0).toUpperCase() + presetCluster.slice(1)}"</CardTitle>
+              <span>{[...clusterInfo?.processorTypes].join(', ')}</span>
+            </CardHeader>
+            <CardBody>
+              <Table borderless>
+                <tr class="py-2">
+                  <td style="font-size:x-large;">{clusterInfo?.runningJobs} Running Jobs</td>
+                  <td colspan="2" style="font-size:x-large;">{clusterInfo?.activeUsers} Active Users</td>
+                </tr>
+                <hr class="my-1"/>
+                <tr class="pt-2">
+                  <td style="font-size: large;">
+                    Flop Rate (<span style="cursor: help;" title="Flops[Any] = (Flops[Double] x 2) + Flops[Single]">Any</span>)
+                  </td>
+                  <td colspan="2" style="font-size: large;">
+                    Memory BW Rate
+                  </td>
+                </tr>
+                <tr class="pb-2">
+                  <td style="font-size:x-large;">
+                    {clusterInfo?.flopRate} 
+                    {clusterInfo?.flopRateUnit}
+                  </td>
+                  <td colspan="2" style="font-size:x-large;">
+                    {clusterInfo?.memBwRate} 
+                    {clusterInfo?.memBwRateUnit}
+                  </td>
+                </tr>
+                <hr class="my-1"/>
+                <tr class="py-2">
+                  <th scope="col">Allocated Nodes</th>
+                  <td style="min-width: 100px;"
+                    ><div class="col">
+                      <Progress
+                        value={clusterInfo?.allocatedNodes}
+                        max={clusterInfo?.totalNodes}
+                      />
+                    </div></td
+                  >
+                  <td
+                    >{clusterInfo?.allocatedNodes} / {clusterInfo?.totalNodes}
+                    Nodes</td
+                  >
+                </tr>
+                <tr class="py-2">
+                  <th scope="col">Allocated Cores</th>
+                  <td style="min-width: 100px;"
+                    ><div class="col">
+                      <Progress
+                        value={clusterInfo?.allocatedCores}
+                        max={clusterInfo?.totalCores}
+                      />
+                    </div></td
+                  >
+                  <td
+                    >{formatNumber(clusterInfo?.allocatedCores)} / {formatNumber(clusterInfo?.totalCores)}
+                    Cores</td
+                  >
+                </tr>
+                {#if clusterInfo?.totalAccs !== 0}
+                  <tr class="py-2">
+                    <th scope="col">Allocated Accelerators</th>
+                    <td style="min-width: 100px;"
+                      ><div class="col">
+                        <Progress
+                          value={clusterInfo?.allocatedAccs}
+                          max={clusterInfo?.totalAccs}
+                        />
+                      </div></td
+                    >
+                    <td
+                      >{clusterInfo?.allocatedAccs} / {clusterInfo?.totalAccs}
+                      Accelerators</td
+                    >
+                  </tr>
+                {/if}
+              </Table>
+            </CardBody>
+          </Card>
+        </Col>
+
+        <Col> <!-- Pie Jobs -->
+          {#if topJobsQuery?.data?.jobsStatistics?.length > 0}
+            <Row cols={{xs:1, md:2}}>
+              <Col class="p-2">
+                <div bind:clientWidth={colWidthJobs}>
+                  <h4 class="text-center">
+                    Top Projects: Jobs
+                  </h4>
+                  <Pie
+                    {useCbColors}
+                    canvasId="hpcpie-jobs-projects"
+                    size={colWidthJobs * 0.75}
+                    sliceLabel={'Jobs'}
+                    quantities={$topJobsQuery.data.jobsStatistics.map(
+                      (tp) => tp['totalJobs'],
+                    )}
+                    entities={$topJobsQuery.data.jobsStatistics.map((tp) => scrambleNames ? scramble(tp.id) : tp.id)}
+                  />
+                </div>
+              </Col>
+              <Col class="p-2">
+                <Table>
+                  <tr class="mb-2">
+                    <th></th>
+                    <th style="padding-left: 0.5rem;">Project</th>
+                    <th>Jobs</th>
+                  </tr>
+                  {#each $topJobsQuery.data.jobsStatistics as tp, i}
+                    <tr>
+                      <td><Icon name="circle-fill" style="color: {legendColors(i)};" /></td>
+                      <td>
+                        <a target="_blank" href="/monitoring/jobs/?cluster={presetCluster}&state=running&project={tp.id}&projectMatch=eq"
+                          >{scrambleNames ? scramble(tp.id) : tp.id}
+                        </a>
+                      </td>
+                      <td>{tp['totalJobs']}</td>
+                    </tr>
+                  {/each}
+                </Table>
+              </Col>
+            </Row>
+          {:else}
+            <Card body color="warning" class="mx-4 my-2"
+              >Cannot render job status: No state data returned for <code>Pie Chart</code></Card
+            >
+          {/if}
+        </Col>
+
+        <Col> <!-- Job Roofline -->
+          <div bind:clientWidth={colWidthRoof}>
+            {#key $statusQuery?.data?.jobsMetricStats}
+              <Roofline
+                useColors={true}
+                allowSizeChange
+                width={colWidthRoof - 10}
+                height={300}
+                subCluster={clusterInfo?.roofData ? clusterInfo.roofData : null}
+                roofData={transformJobsStatsToData($statusQuery?.data?.jobsMetricStats)}
+                jobsData={transformJobsStatsToInfo($statusQuery?.data?.jobsMetricStats)}
+              />
+            {/key}
+          </div>
+        </Col>
+
+        <Col> <!-- Total Cluster Metric in Time SUMS-->
+          <div bind:clientWidth={colWidthTotals}>
+            <DoubleMetric
+              width={colWidthTotals}
+              timestep={$statusQuery?.data?.clusterMetrics[0]?.timestep || 60}
+              numNodes={$statusQuery?.data?.clusterMetrics?.nodeCount || 0}
+              metricData={$statusQuery?.data?.clusterMetrics?.metrics || []}
+              cluster={presetCluster}
+              fixLinewidth={2}
+            />
+          </div>
+        </Col>
+
+        <Col> <!-- Stacked SchedState -->
+          <div bind:clientWidth={colWidthStacked1}>
+            {#key $statesTimed?.data?.nodeStates}
+              <Stacked
+                data={$statesTimed?.data?.nodeStates}
+                width={colWidthStacked1 * 0.95}
+                xlabel="Time"
+                ylabel="Nodes"
+                yunit = "#Count"
+                title = "Node States"
+                stateType = "Node"
+              />
+            {/key}
+          </div>
+        </Col>
+
+        <Col> <!-- Stacked Healthstate -->
+          <div bind:clientWidth={colWidthStacked2}>
+            {#key $statesTimed?.data?.healthStates}
+              <Stacked
+                data={$statesTimed?.data?.healthStates}
+                width={colWidthStacked2 * 0.95}
+                xlabel="Time"
+                ylabel="Nodes"
+                yunit = "#Count"
+                title = "Health States"
+                stateType = "Health"
+              />
+            {/key}
+          </div>
+        </Col>
+      </Row>
+    {/if}
+  </CardBody>
+</Card>

web/frontend/src/status/dashdetails/StatisticsDash.svelte

@@ -22,11 +22,11 @@
   } from "@urql/svelte";
   import {
     convert2uplot,
-  } from "../generic/utils.js";
+  } from "../../generic/utils.js";
-  import PlotGrid from "../generic/PlotGrid.svelte";
+  import PlotGrid from "../../generic/PlotGrid.svelte";
-  import Histogram from "../generic/plots/Histogram.svelte";
+  import Histogram from "../../generic/plots/Histogram.svelte";
-  import HistogramSelection from "../generic/select/HistogramSelection.svelte";
+  import HistogramSelection from "../../generic/select/HistogramSelection.svelte";
-  import Refresher from "../generic/helper/Refresher.svelte";
+  import Refresher from "../../generic/helper/Refresher.svelte";
 
   /* Svelte 5 Props */
   let {

web/frontend/src/status/dashdetails/StatusDash.svelte

@@ -22,12 +22,12 @@
     gql,
     getContextClient,
   } from "@urql/svelte";
-  import { formatDurationTime } from "../generic/units.js";
+  import { formatDurationTime } from "../../generic/units.js";
-  import Refresher from "../generic/helper/Refresher.svelte";
+  import Refresher from "../../generic/helper/Refresher.svelte";
-  import TimeSelection from "../generic/select/TimeSelection.svelte";
+  import TimeSelection from "../../generic/select/TimeSelection.svelte";
-  import Roofline from "../generic/plots/Roofline.svelte";
+  import Roofline from "../../generic/plots/Roofline.svelte";
-  import Pie, { colors } from "../generic/plots/Pie.svelte";
+  import Pie, { colors } from "../../generic/plots/Pie.svelte";
-  import Stacked from "../generic/plots/Stacked.svelte";
+  import Stacked from "../../generic/plots/Stacked.svelte";
 
   /* Svelte 5 Props */
   let {
@@ -402,7 +402,7 @@
         to = new Date(Date.now());
 
         if (interval) stackedFrom += Math.floor(interval / 1000);
-        else stackedFrom += 1 // Workaround: TineSelection not linked, just trigger new data on manual refresh
+        else stackedFrom += 1 // Workaround: TimeSelection not linked, just trigger new data on manual refresh
       }}
     />
   </Col>

web/frontend/src/status/dashdetails/UsageDash.svelte

@@ -27,10 +27,10 @@
     scramble,
     scrambleNames,
     convert2uplot,
-  } from "../generic/utils.js";
+  } from "../../generic/utils.js";
-  import Pie, { colors } from "../generic/plots/Pie.svelte";
+  import Pie, { colors } from "../../generic/plots/Pie.svelte";
-  import Histogram from "../generic/plots/Histogram.svelte";
+  import Histogram from "../../generic/plots/Histogram.svelte";
-  import Refresher from "../generic/helper/Refresher.svelte";
+  import Refresher from "../../generic/helper/Refresher.svelte";
 
   /* Svelte 5 Props */
   let {

web/templates/base.tmpl

@@ -23,34 +23,49 @@
         </script>
     </head>
     <body  class="site">
-        {{block "navigation" .}}
+        {{if eq .Infos.displayType "PUBLIC"}}
-            <header id="svelte-header"></header>
+            <main>
-        {{end}}
+                <div class="container">
+                    {{block "content-public" .}}
+                        Whoops, you should not see this... [PUBLIC]
+                    {{end}}
+                </div>
+            </main>
 
-        <main class="site-content">
+            {{block "javascript-public" .}}
-            <div class="container">
+                Whoops, you should not see this... [JS PUBLIC]
-                {{block "content" .}}
+            {{end}}
-                    Whoops, you should not see this...
-                {{end}}
-            </div>
-        </main>
 
-        {{block "footer" .}}
+        {{else}}
-            <footer class="site-footer bg-light">
+            {{block "navigation" .}}
-                <ul class="footer-list">
+                <header id="svelte-header"></header>
-                    <li class="footer-list-item"><a class="link-secondary fs-5" href="/imprint" title="Imprint" rel="nofollow">Imprint</a></li>
+            {{end}}
-                    <li class="footer-list-item"><a class="link-secondary fs-5" href="/privacy" title="Privacy Policy" rel="nofollow">Privacy Policy</a></li>
-                </ul>
-                <ul class="build-list">
-                  <li class="build-list-item">Version {{ .Build.Version }}</li>
-                  <li class="build-list-item">Hash {{ .Build.Hash }}</li>
-                  <li class="build-list-item">Built {{ .Build.Buildtime }}</li>
-                </ul>
-            </footer>
-        {{end}}
 
-        {{block "javascript" .}}
+            <main class="site-content">
-            <script src='/build/header.js'></script>
+                <div class="container">
+                    {{block "content" .}}
+                        Whoops, you should not see this... [MAIN]
+                    {{end}}
+                </div>
+            </main>
+
+            {{block "footer" .}}
+                <footer class="site-footer bg-light">
+                    <ul class="footer-list">
+                        <li class="footer-list-item"><a class="link-secondary fs-5" href="/imprint" title="Imprint" rel="nofollow">Imprint</a></li>
+                        <li class="footer-list-item"><a class="link-secondary fs-5" href="/privacy" title="Privacy Policy" rel="nofollow">Privacy Policy</a></li>
+                    </ul>
+                    <ul class="build-list">
+                    <li class="build-list-item">Version {{ .Build.Version }}</li>
+                    <li class="build-list-item">Hash {{ .Build.Hash }}</li>
+                    <li class="build-list-item">Built {{ .Build.Buildtime }}</li>
+                    </ul>
+                </footer>
+            {{end}}
+
+            {{block "javascript" .}}
+                <script src='/build/header.js'></script>
+            {{end}}
         {{end}}
     </body>
 </html>

web/templates/monitoring/dashboard.tmpl

@@ -0,0 +1,14 @@
+{{define "content-public"}}
+    <div id="svelte-app"></div>
+{{end}}
+
+{{define "stylesheets"}}
+    <link rel='stylesheet' href='/build/dashpublic.css'>
+{{end}}
+{{define "javascript-public"}}
+    <script>
+        const presetCluster = {{ .Infos.cluster }};
+        const clusterCockpitConfig = {{ .Config }};
+    </script>
+    <script src='/build/dashpublic.js'></script>
+{{end}}

web/templates/monitoring/status.tmpl

@@ -8,6 +8,7 @@
 {{define "javascript"}}
     <script>
         const infos = {{ .Infos }};
+        const displayType = {{ .Infos.displayType }};
         const clusterCockpitConfig = {{ .Config }};
     </script>
     <script src='/build/status.js'></script>