     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package main provides the entry point for the ClusterCockpit backend server.
     7→// This file contains HTTP server setup, routing configuration, and
     8→// authentication middleware integration.
     9→package main
    10→
    11→import (
    12→	"context"
    13→	"crypto/tls"
    14→	"encoding/json"
    15→	"errors"
    16→	"fmt"
    17→	"net"
    18→	"net/http"
    19→	"os"
    20→	"strings"
    21→	"time"
    22→
    23→	"github.com/99designs/gqlgen/graphql/handler"
    24→	"github.com/99designs/gqlgen/graphql/handler/transport"
    25→	"github.com/99designs/gqlgen/graphql/playground"
    26→	"github.com/ClusterCockpit/cc-backend/internal/api"
    27→	"github.com/ClusterCockpit/cc-backend/internal/archiver"
    28→	"github.com/ClusterCockpit/cc-backend/internal/auth"
    29→	"github.com/ClusterCockpit/cc-backend/internal/config"
    30→	"github.com/ClusterCockpit/cc-backend/internal/graph"
    31→	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
    32→	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
    33→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    34→	"github.com/ClusterCockpit/cc-backend/web"
    35→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    36→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    37→	"github.com/ClusterCockpit/cc-lib/v2/runtime"
    38→	"github.com/go-chi/chi/v5"
    39→	"github.com/go-chi/chi/v5/middleware"
    40→	"github.com/go-chi/cors"
    41→	httpSwagger "github.com/swaggo/http-swagger"
    42→)
    43→
    44→var buildInfo web.Build
    45→
    46→// Environment variable names
    47→const (
    48→	envDebug = "DEBUG"
    49→)
    50→
    51→// Server encapsulates the HTTP server state and dependencies
    52→type Server struct {
    53→	router        chi.Router
    54→	server        *http.Server
    55→	restAPIHandle *api.RestAPI
    56→	natsAPIHandle *api.NatsAPI
    57→}
    58→
    59→func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) {
    60→	rw.Header().Add("Content-Type", "application/json")
    61→	rw.WriteHeader(http.StatusUnauthorized)
    62→	json.NewEncoder(rw).Encode(map[string]string{
    63→		"status": http.StatusText(http.StatusUnauthorized),
    64→		"error":  err.Error(),
    65→	})
    66→}
    67→
    68→// NewServer creates and initializes a new Server instance
    69→func NewServer(version, commit, buildDate string) (*Server, error) {
    70→	buildInfo = web.Build{Version: version, Hash: commit, Buildtime: buildDate}
    71→
    72→	s := &Server{
    73→		router: chi.NewRouter(),
    74→	}
    75→
    76→	if err := s.init(); err != nil {
    77→		return nil, err
    78→	}
    79→
    80→	return s, nil
    81→}
    82→
    83→func (s *Server) init() error {
    84→	// Setup the http.Handler/Router used by the server
    85→	graph.Init()
    86→	resolver := graph.GetResolverInstance()
    87→	graphQLServer := handler.New(
    88→		generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))
    89→
    90→	graphQLServer.AddTransport(transport.POST{})
    91→
    92→	if os.Getenv(envDebug) != "1" {
    93→		// Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed.
    94→		// The problem with this is that then, no more stacktrace is printed to stderr.
    95→		graphQLServer.SetRecoverFunc(func(ctx context.Context, err any) error {
    96→			switch e := err.(type) {
    97→			case string:
    98→				return fmt.Errorf("MAIN > Panic: %s", e)
    99→			case error:
   100→				return fmt.Errorf("MAIN > Panic caused by: %s", e.Error())
   101→			}
   102→
   103→			return errors.New("MAIN > Internal server error (panic)")
   104→		})
   105→	}
   106→
   107→	authHandle := auth.GetAuthInstance()
   108→
   109→	// Middleware must be defined before routes in chi
   110→	s.router.Use(func(next http.Handler) http.Handler {
   111→		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
   112→			start := time.Now()
   113→			ww := middleware.NewWrapResponseWriter(rw, r.ProtoMajor)
   114→			next.ServeHTTP(ww, r)
   115→			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
   116→				r.Method, r.URL.RequestURI(),
   117→				ww.Status(), float32(ww.BytesWritten())/1024,
   118→				time.Since(start).Milliseconds())
   119→		})
   120→	})
   121→	s.router.Use(middleware.Compress(5))
   122→	s.router.Use(middleware.Recoverer)
   123→	s.router.Use(cors.Handler(cors.Options{
   124→		AllowCredentials: true,
   125→		AllowedHeaders:   []string{"X-Requested-With", "Content-Type", "Authorization", "Origin"},
   126→		AllowedMethods:   []string{"GET", "POST", "HEAD", "OPTIONS"},
   127→		AllowedOrigins:   []string{"*"},
   128→	}))
   129→
   130→	s.restAPIHandle = api.New()
   131→
   132→	info := map[string]any{}
   133→	info["hasOpenIDConnect"] = false
   134→
   135→	if auth.Keys.OpenIDConfig != nil {
   136→		openIDConnect := auth.NewOIDC(authHandle)
   137→		openIDConnect.RegisterEndpoints(s.router)
   138→		info["hasOpenIDConnect"] = true
   139→	}
   140→
   141→	s.router.Get("/login", func(rw http.ResponseWriter, r *http.Request) {
   142→		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
   143→		cclog.Debugf("##%v##", info)
   144→		web.RenderTemplate(rw, "login.tmpl", &web.Page{Title: "Login", Build: buildInfo, Infos: info})
   145→	})
   146→	s.router.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
   147→		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
   148→		web.RenderTemplate(rw, "imprint.tmpl", &web.Page{Title: "Imprint", Build: buildInfo})
   149→	})
   150→	s.router.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) {
   151→		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
   152→		web.RenderTemplate(rw, "privacy.tmpl", &web.Page{Title: "Privacy", Build: buildInfo})
   153→	})
   154→
   155→	if !config.Keys.DisableAuthentication {
   156→		// Create login failure handler (used by both /login and /jwt-login)
   157→		loginFailureHandler := func(rw http.ResponseWriter, r *http.Request, err error) {
   158→			rw.Header().Add("Content-Type", "text/html; charset=utf-8")
   159→			rw.WriteHeader(http.StatusUnauthorized)
   160→			web.RenderTemplate(rw, "login.tmpl", &web.Page{
   161→				Title:   "Login failed - ClusterCockpit",
   162→				MsgType: "alert-warning",
   163→				Message: err.Error(),
   164→				Build:   buildInfo,
   165→				Infos:   info,
   166→			})
   167→		}
   168→
   169→		s.router.Post("/login", authHandle.Login(loginFailureHandler).ServeHTTP)
   170→		s.router.HandleFunc("/jwt-login", authHandle.Login(loginFailureHandler).ServeHTTP)
   171→
   172→		s.router.Post("/logout", authHandle.Logout(
   173→			http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
   174→				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
   175→				rw.WriteHeader(http.StatusOK)
   176→				web.RenderTemplate(rw, "login.tmpl", &web.Page{
   177→					Title:   "Bye - ClusterCockpit",
   178→					MsgType: "alert-info",
   179→					Message: "Logout successful",
   180→					Build:   buildInfo,
   181→					Infos:   info,
   182→				})
   183→			})).ServeHTTP)
   184→	}
   185→
   186→	if flagDev {
   187→		s.router.Handle("/playground", playground.Handler("GraphQL playground", "/query"))
   188→		s.router.Get("/swagger/*", httpSwagger.Handler(
   189→			httpSwagger.URL("http://"+config.Keys.Addr+"/swagger/doc.json")))
   190→	}
   191→
   192→	// Secured routes (require authentication)
   193→	s.router.Group(func(secured chi.Router) {
   194→		if !config.Keys.DisableAuthentication {
   195→			secured.Use(func(next http.Handler) http.Handler {
   196→				return authHandle.Auth(
   197→					next,
   198→					func(rw http.ResponseWriter, r *http.Request, err error) {
   199→						rw.WriteHeader(http.StatusUnauthorized)
   200→						web.RenderTemplate(rw, "login.tmpl", &web.Page{
   201→							Title:    "Authentication failed - ClusterCockpit",
   202→							MsgType:  "alert-danger",
   203→							Message:  err.Error(),
   204→							Build:    buildInfo,
   205→							Infos:    info,
   206→							Redirect: r.RequestURI,
   207→						})
   208→					})
   209→			})
   210→		}
   211→
   212→		secured.Handle("/query", graphQLServer)
   213→
   214→		secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
   215→			routerConfig.HandleSearchBar(rw, r, buildInfo)
   216→		})
   217→
   218→		routerConfig.SetupRoutes(secured, buildInfo)
   219→	})
   220→
   221→	// API routes (JWT token auth)
   222→	s.router.Route("/api", func(apiRouter chi.Router) {
   223→		// Main API routes with API auth
   224→		apiRouter.Group(func(securedapi chi.Router) {
   225→			if !config.Keys.DisableAuthentication {
   226→				securedapi.Use(func(next http.Handler) http.Handler {
   227→					return authHandle.AuthAPI(next, onFailureResponse)
   228→				})
   229→			}
   230→			s.restAPIHandle.MountAPIRoutes(securedapi)
   231→		})
   232→
   233→		// Metric store API routes with separate auth
   234→		apiRouter.Group(func(metricstoreapi chi.Router) {
   235→			if !config.Keys.DisableAuthentication {
   236→				metricstoreapi.Use(func(next http.Handler) http.Handler {
   237→					return authHandle.AuthMetricStoreAPI(next, onFailureResponse)
   238→				})
   239→			}
   240→			s.restAPIHandle.MountMetricStoreAPIRoutes(metricstoreapi)
   241→		})
   242→	})
   243→
   244→	// User API routes
   245→	s.router.Route("/userapi", func(userapi chi.Router) {
   246→		if !config.Keys.DisableAuthentication {
   247→			userapi.Use(func(next http.Handler) http.Handler {
   248→				return authHandle.AuthUserAPI(next, onFailureResponse)
   249→			})
   250→		}
   251→		s.restAPIHandle.MountUserAPIRoutes(userapi)
   252→	})
   253→
   254→	// Config API routes (uses Group with full paths to avoid shadowing
   255→	// the /config page route that is registered in the secured group)
   256→	s.router.Group(func(configapi chi.Router) {
   257→		if !config.Keys.DisableAuthentication {
   258→			configapi.Use(func(next http.Handler) http.Handler {
   259→				return authHandle.AuthConfigAPI(next, onFailureResponse)
   260→			})
   261→		}
   262→		s.restAPIHandle.MountConfigAPIRoutes(configapi)
   263→	})
   264→
   265→	// Frontend API routes
   266→	s.router.Route("/frontend", func(frontendapi chi.Router) {
   267→		if !config.Keys.DisableAuthentication {
   268→			frontendapi.Use(func(next http.Handler) http.Handler {
   269→				return authHandle.AuthFrontendAPI(next, onFailureResponse)
   270→			})
   271→		}
   272→		s.restAPIHandle.MountFrontendAPIRoutes(frontendapi)
   273→	})
   274→
   275→	if config.Keys.APISubjects != nil {
   276→		s.natsAPIHandle = api.NewNatsAPI()
   277→		if err := s.natsAPIHandle.StartSubscriptions(); err != nil {
   278→			return fmt.Errorf("starting NATS subscriptions: %w", err)
   279→		}
   280→	}
   281→
   282→	// 404 handler for pages and API routes
   283→	notFoundHandler := func(rw http.ResponseWriter, r *http.Request) {
   284→		if strings.HasPrefix(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/userapi/") ||
   285→			strings.HasPrefix(r.URL.Path, "/frontend/") || strings.HasPrefix(r.URL.Path, "/config/") {
   286→			rw.Header().Set("Content-Type", "application/json")
   287→			rw.WriteHeader(http.StatusNotFound)
   288→			json.NewEncoder(rw).Encode(map[string]string{
   289→				"status": "Resource not found",
   290→				"error":  "the requested endpoint does not exist",
   291→			})
   292→			return
   293→		}
   294→		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
   295→		rw.WriteHeader(http.StatusNotFound)
   296→		web.RenderTemplate(rw, "404.tmpl", &web.Page{
   297→			Title: "Page Not Found",
   298→			Build: buildInfo,
   299→		})
   300→	}
   301→
   302→	// Set NotFound on the router so chi uses it for all unmatched routes,
   303→	// including those under subrouters like /api, /userapi, /frontend, etc.
   304→	s.router.NotFound(notFoundHandler)
   305→
   306→	if config.Keys.EmbedStaticFiles {
   307→		if i, err := os.Stat("./var/img"); err == nil {
   308→			if i.IsDir() {
   309→				cclog.Info("Use local directory for static images")
   310→				s.router.Handle("/img/*", http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
   311→			}
   312→		}
   313→		fileServer := http.StripPrefix("/", web.ServeFiles())
   314→		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
   315→			if web.StaticFileExists(r.URL.Path) {
   316→				fileServer.ServeHTTP(rw, r)
   317→				return
   318→			}
   319→			notFoundHandler(rw, r)
   320→		}))
   321→	} else {
   322→		staticDir := http.Dir(config.Keys.StaticFiles)
   323→		fileServer := http.FileServer(staticDir)
   324→		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
   325→			f, err := staticDir.Open(r.URL.Path)
   326→			if err == nil {
   327→				f.Close()
   328→				fileServer.ServeHTTP(rw, r)
   329→				return
   330→			}
   331→			notFoundHandler(rw, r)
   332→		}))
   333→	}
   334→
   335→	return nil
   336→}
   337→
   338→// Server timeout defaults (in seconds)
   339→const (
   340→	defaultReadTimeout  = 20
   341→	defaultWriteTimeout = 20
   342→)
   343→
   344→func (s *Server) Start(ctx context.Context) error {
   345→	// Use configurable timeouts with defaults
   346→	readTimeout := time.Duration(defaultReadTimeout) * time.Second
   347→	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second
   348→
   349→	s.server = &http.Server{
   350→		ReadTimeout:  readTimeout,
   351→		WriteTimeout: writeTimeout,
   352→		Handler:      s.router,
   353→		Addr:         config.Keys.Addr,
   354→	}
   355→
   356→	// Start http or https server
   357→	listener, err := net.Listen("tcp", config.Keys.Addr)
   358→	if err != nil {
   359→		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Addr, err)
   360→	}
   361→
   362→	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHTTPTo != "" {
   363→		go func() {
   364→			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHTTPTo, http.StatusMovedPermanently))
   365→		}()
   366→	}
   367→
   368→	if config.Keys.HTTPSCertFile != "" && config.Keys.HTTPSKeyFile != "" {
   369→		cert, err := tls.LoadX509KeyPair(
   370→			config.Keys.HTTPSCertFile, config.Keys.HTTPSKeyFile)
   371→		if err != nil {
   372→			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
   373→		}
   374→		listener = tls.NewListener(listener, &tls.Config{
   375→			Certificates: []tls.Certificate{cert},
   376→			CipherSuites: []uint16{
   377→				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
   378→				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
   379→			},
   380→			MinVersion:               tls.VersionTLS12,
   381→			PreferServerCipherSuites: true,
   382→		})
   383→		cclog.Infof("HTTPS server listening at %s...", config.Keys.Addr)
   384→	} else {
   385→		cclog.Infof("HTTP server listening at %s...", config.Keys.Addr)
   386→	}
   387→	//
   388→	// Because this program will want to bind to a privileged port (like 80), the listener must
   389→	// be established first, then the user can be changed, and after that,
   390→	// the actual http server can be started.
   391→	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
   392→		return fmt.Errorf("dropping privileges: %w", err)
   393→	}
   394→
   395→	// Handle context cancellation for graceful shutdown
   396→	go func() {
   397→		<-ctx.Done()
   398→		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
   399→		defer cancel()
   400→		if err := s.server.Shutdown(shutdownCtx); err != nil {
   401→			cclog.Errorf("Server shutdown error: %v", err)
   402→		}
   403→	}()
   404→
   405→	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
   406→		return fmt.Errorf("server failed: %w", err)
   407→	}
   408→	return nil
   409→}
   410→
   411→func (s *Server) Shutdown(ctx context.Context) {
   412→	// Create a shutdown context with timeout
   413→	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
   414→	defer cancel()
   415→
   416→	nc := nats.GetClient()
   417→	if nc != nil {
   418→		nc.Close()
   419→	}
   420→
   421→	// First shut down the server gracefully (waiting for all ongoing requests)
   422→	if err := s.server.Shutdown(shutdownCtx); err != nil {
   423→		cclog.Errorf("Server shutdown error: %v", err)
   424→	}
   425→
   426→	// Archive all the metric store data
   427→	ms := metricstore.GetMemoryStore()
   428→
   429→	if ms != nil {
   430→		metricstore.Shutdown()
   431→	}
   432→
   433→	// Shutdown archiver with 10 second timeout for fast shutdown
   434→	if err := archiver.Shutdown(10 * time.Second); err != nil {
   435→		cclog.Warnf("Archiver shutdown: %v", err)
   436→	}
   437→}
   438→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package main provides the entry point for the ClusterCockpit backend server.
// This file contains HTTP server setup, routing configuration, and
// authentication middleware integration.
package main

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/99designs/gqlgen/graphql/handler"
	"github.com/99designs/gqlgen/graphql/handler/transport"
	"github.com/99designs/gqlgen/graphql/playground"
	"github.com/ClusterCockpit/cc-backend/internal/api"
	"github.com/ClusterCockpit/cc-backend/internal/archiver"
	"github.com/ClusterCockpit/cc-backend/internal/auth"
	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/graph"
	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
	"github.com/ClusterCockpit/cc-backend/web"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/nats"
	"github.com/ClusterCockpit/cc-lib/v2/runtime"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/cors"
	httpSwagger "github.com/swaggo/http-swagger"
)

var buildInfo web.Build

// Environment variable names
const (
	envDebug = "DEBUG"
)

// Server encapsulates the HTTP server state and dependencies
type Server struct {
	router        chi.Router
	server        *http.Server
	restAPIHandle *api.RestAPI
	natsAPIHandle *api.NatsAPI
}

func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) {
	rw.Header().Add("Content-Type", "application/json")
	rw.WriteHeader(http.StatusUnauthorized)
	json.NewEncoder(rw).Encode(map[string]string{
		"status": http.StatusText(http.StatusUnauthorized),
		"error":  err.Error(),
	})
}

// NewServer creates and initializes a new Server instance
func NewServer(version, commit, buildDate string) (*Server, error) {
	buildInfo = web.Build{Version: version, Hash: commit, Buildtime: buildDate}

	s := &Server{
		router: chi.NewRouter(),
	}

	if err := s.init(); err != nil {
		return nil, err
	}

	return s, nil
}

func (s *Server) init() error {
	// Setup the http.Handler/Router used by the server
	graph.Init()
	resolver := graph.GetResolverInstance()
	graphQLServer := handler.New(
		generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))

	graphQLServer.AddTransport(transport.POST{})

	if os.Getenv(envDebug) != "1" {
		// Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed.
		// The problem with this is that then, no more stacktrace is printed to stderr.
		graphQLServer.SetRecoverFunc(func(ctx context.Context, err any) error {
			switch e := err.(type) {
			case string:
				return fmt.Errorf("MAIN > Panic: %s", e)
			case error:
				return fmt.Errorf("MAIN > Panic caused by: %s", e.Error())
			}

			return errors.New("MAIN > Internal server error (panic)")
		})
	}

	authHandle := auth.GetAuthInstance()

	// Middleware must be defined before routes in chi
	s.router.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			start := time.Now()
			ww := middleware.NewWrapResponseWriter(rw, r.ProtoMajor)
			next.ServeHTTP(ww, r)
			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
				r.Method, r.URL.RequestURI(),
				ww.Status(), float32(ww.BytesWritten())/1024,
				time.Since(start).Milliseconds())
		})
	})
	s.router.Use(middleware.Compress(5))
	s.router.Use(middleware.Recoverer)
	s.router.Use(cors.Handler(cors.Options{
		AllowCredentials: true,
		AllowedHeaders:   []string{"X-Requested-With", "Content-Type", "Authorization", "Origin"},
		AllowedMethods:   []string{"GET", "POST", "HEAD", "OPTIONS"},
		AllowedOrigins:   []string{"*"},
	}))

	s.restAPIHandle = api.New()

	info := map[string]any{}
	info["hasOpenIDConnect"] = false

	if auth.Keys.OpenIDConfig != nil {
		openIDConnect := auth.NewOIDC(authHandle)
		openIDConnect.RegisterEndpoints(s.router)
		info["hasOpenIDConnect"] = true
	}

	s.router.Get("/login", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		cclog.Debugf("##%v##", info)
		web.RenderTemplate(rw, "login.tmpl", &web.Page{Title: "Login", Build: buildInfo, Infos: info})
	})
	s.router.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		web.RenderTemplate(rw, "imprint.tmpl", &web.Page{Title: "Imprint", Build: buildInfo})
	})
	s.router.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
		web.RenderTemplate(rw, "privacy.tmpl", &web.Page{Title: "Privacy", Build: buildInfo})
	})

	if !config.Keys.DisableAuthentication {
		// Create login failure handler (used by both /login and /jwt-login)
		loginFailureHandler := func(rw http.ResponseWriter, r *http.Request, err error) {
			rw.Header().Add("Content-Type", "text/html; charset=utf-8")
			rw.WriteHeader(http.StatusUnauthorized)
			web.RenderTemplate(rw, "login.tmpl", &web.Page{
				Title:   "Login failed - ClusterCockpit",
				MsgType: "alert-warning",
				Message: err.Error(),
				Build:   buildInfo,
				Infos:   info,
			})
		}

		s.router.Post("/login", authHandle.Login(loginFailureHandler).ServeHTTP)
		s.router.HandleFunc("/jwt-login", authHandle.Login(loginFailureHandler).ServeHTTP)

		s.router.Post("/logout", authHandle.Logout(
			http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
				rw.WriteHeader(http.StatusOK)
				web.RenderTemplate(rw, "login.tmpl", &web.Page{
					Title:   "Bye - ClusterCockpit",
					MsgType: "alert-info",
					Message: "Logout successful",
					Build:   buildInfo,
					Infos:   info,
				})
			})).ServeHTTP)
	}

	if flagDev {
		s.router.Handle("/playground", playground.Handler("GraphQL playground", "/query"))
		s.router.Get("/swagger/*", httpSwagger.Handler(
			httpSwagger.URL("http://"+config.Keys.Addr+"/swagger/doc.json")))
	}

	// Secured routes (require authentication)
	s.router.Group(func(secured chi.Router) {
		if !config.Keys.DisableAuthentication {
			secured.Use(func(next http.Handler) http.Handler {
				return authHandle.Auth(
					next,
					func(rw http.ResponseWriter, r *http.Request, err error) {
						rw.WriteHeader(http.StatusUnauthorized)
						web.RenderTemplate(rw, "login.tmpl", &web.Page{
							Title:    "Authentication failed - ClusterCockpit",
							MsgType:  "alert-danger",
							Message:  err.Error(),
							Build:    buildInfo,
							Infos:    info,
							Redirect: r.RequestURI,
						})
					})
			})
		}

		secured.Handle("/query", graphQLServer)

		secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
			routerConfig.HandleSearchBar(rw, r, buildInfo)
		})

		routerConfig.SetupRoutes(secured, buildInfo)
	})

	// API routes (JWT token auth)
	s.router.Route("/api", func(apiRouter chi.Router) {
		// Main API routes with API auth
		apiRouter.Group(func(securedapi chi.Router) {
			if !config.Keys.DisableAuthentication {
				securedapi.Use(func(next http.Handler) http.Handler {
					return authHandle.AuthAPI(next, onFailureResponse)
				})
			}
			s.restAPIHandle.MountAPIRoutes(securedapi)
		})

		// Metric store API routes with separate auth
		apiRouter.Group(func(metricstoreapi chi.Router) {
			if !config.Keys.DisableAuthentication {
				metricstoreapi.Use(func(next http.Handler) http.Handler {
					return authHandle.AuthMetricStoreAPI(next, onFailureResponse)
				})
			}
			s.restAPIHandle.MountMetricStoreAPIRoutes(metricstoreapi)
		})
	})

	// User API routes
	s.router.Route("/userapi", func(userapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			userapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthUserAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountUserAPIRoutes(userapi)
	})

	// Config API routes (uses Group with full paths to avoid shadowing
	// the /config page route that is registered in the secured group)
	s.router.Group(func(configapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			configapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthConfigAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountConfigAPIRoutes(configapi)
	})

	// Frontend API routes
	s.router.Route("/frontend", func(frontendapi chi.Router) {
		if !config.Keys.DisableAuthentication {
			frontendapi.Use(func(next http.Handler) http.Handler {
				return authHandle.AuthFrontendAPI(next, onFailureResponse)
			})
		}
		s.restAPIHandle.MountFrontendAPIRoutes(frontendapi)
	})

	if config.Keys.APISubjects != nil {
		s.natsAPIHandle = api.NewNatsAPI()
		if err := s.natsAPIHandle.StartSubscriptions(); err != nil {
			return fmt.Errorf("starting NATS subscriptions: %w", err)
		}
	}

	// 404 handler for pages and API routes
	notFoundHandler := func(rw http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/userapi/") ||
			strings.HasPrefix(r.URL.Path, "/frontend/") || strings.HasPrefix(r.URL.Path, "/config/") {
			rw.Header().Set("Content-Type", "application/json")
			rw.WriteHeader(http.StatusNotFound)
			json.NewEncoder(rw).Encode(map[string]string{
				"status": "Resource not found",
				"error":  "the requested endpoint does not exist",
			})
			return
		}
		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
		rw.WriteHeader(http.StatusNotFound)
		web.RenderTemplate(rw, "404.tmpl", &web.Page{
			Title: "Page Not Found",
			Build: buildInfo,
		})
	}

	// Set NotFound on the router so chi uses it for all unmatched routes,
	// including those under subrouters like /api, /userapi, /frontend, etc.
	s.router.NotFound(notFoundHandler)

	if config.Keys.EmbedStaticFiles {
		if i, err := os.Stat("./var/img"); err == nil {
			if i.IsDir() {
				cclog.Info("Use local directory for static images")
				s.router.Handle("/img/*", http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
			}
		}
		fileServer := http.StripPrefix("/", web.ServeFiles())
		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			if web.StaticFileExists(r.URL.Path) {
				fileServer.ServeHTTP(rw, r)
				return
			}
			notFoundHandler(rw, r)
		}))
	} else {
		staticDir := http.Dir(config.Keys.StaticFiles)
		fileServer := http.FileServer(staticDir)
		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
			f, err := staticDir.Open(r.URL.Path)
			if err == nil {
				f.Close()
				fileServer.ServeHTTP(rw, r)
				return
			}
			notFoundHandler(rw, r)
		}))
	}

	return nil
}

// Server timeout defaults (in seconds)
const (
	defaultReadTimeout  = 20
	defaultWriteTimeout = 20
)

func (s *Server) Start(ctx context.Context) error {
	// Use configurable timeouts with defaults
	readTimeout := time.Duration(defaultReadTimeout) * time.Second
	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second

	s.server = &http.Server{
		ReadTimeout:  readTimeout,
		WriteTimeout: writeTimeout,
		Handler:      s.router,
		Addr:         config.Keys.Addr,
	}

	// Start http or https server
	listener, err := net.Listen("tcp", config.Keys.Addr)
	if err != nil {
		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Addr, err)
	}

	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHTTPTo != "" {
		go func() {
			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHTTPTo, http.StatusMovedPermanently))
		}()
	}

	if config.Keys.HTTPSCertFile != "" && config.Keys.HTTPSKeyFile != "" {
		cert, err := tls.LoadX509KeyPair(
			config.Keys.HTTPSCertFile, config.Keys.HTTPSKeyFile)
		if err != nil {
			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
		}
		listener = tls.NewListener(listener, &tls.Config{
			Certificates: []tls.Certificate{cert},
			CipherSuites: []uint16{
				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			},
			MinVersion:               tls.VersionTLS12,
			PreferServerCipherSuites: true,
		})
		cclog.Infof("HTTPS server listening at %s...", config.Keys.Addr)
	} else {
		cclog.Infof("HTTP server listening at %s...", config.Keys.Addr)
	}
	//
	// Because this program will want to bind to a privileged port (like 80), the listener must
	// be established first, then the user can be changed, and after that,
	// the actual http server can be started.
	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
		return fmt.Errorf("dropping privileges: %w", err)
	}

	// Handle context cancellation for graceful shutdown
	go func() {
		<-ctx.Done()
		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		if err := s.server.Shutdown(shutdownCtx); err != nil {
			cclog.Errorf("Server shutdown error: %v", err)
		}
	}()

	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
		return fmt.Errorf("server failed: %w", err)
	}
	return nil
}

func (s *Server) Shutdown(ctx context.Context) {
	// Create a shutdown context with timeout
	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	nc := nats.GetClient()
	if nc != nil {
		nc.Close()
	}

	// First shut down the server gracefully (waiting for all ongoing requests)
	if err := s.server.Shutdown(shutdownCtx); err != nil {
		cclog.Errorf("Server shutdown error: %v", err)
	}

	// Archive all the metric store data
	ms := metricstore.GetMemoryStore()

	if ms != nil {
		metricstore.Shutdown()
	}

	// Shutdown archiver with 10 second timeout for fast shutdown
	if err := archiver.Shutdown(10 * time.Second); err != nil {
		cclog.Warnf("Archiver shutdown: %v", err)
	}
}


1→
    10→
    11→
   198→
   199→
   200→<Row class="justify-content-between">
   201→  <Col class="mb-2 mb-md-0" xs="12" md="5" lg="4" xl="3">
   202→    <InputGroup>
   203→      <InputGroupText>
   204→        <Icon name="bar-chart-line-fill" />
   205→      </InputGroupText>
   206→      <InputGroupText>Duration Bin Size</InputGroupText>
   207→      <Input type="select" bind:value={numDurationBins}>
   208→        {#each durationBinOptions as dbin}
   209→          <option value={dbin}>{dbin}</option>
   210→        {/each}
   211→      </Input>
   212→    </InputGroup>
   213→  </Col>
   214→  <Col xs="12" md="5" lg="4" xl="3">
   215→    <Refresher
   216→      initially={120}
   217→      onRefresh={() => {
   218→        pagingState = { page: 1, itemsPerPage: 10 };
   219→        selectedHistograms = [...$state.snapshot(selectedHistograms)];
   220→      }}
   221→    />
   222→  </Col>
   223→</Row>
   224→
   225→

   226→
   227→
   228→{#if $topStatsQuery?.fetching || $nodeStatusQuery?.fetching}
   229→  <Spinner />
   230→{:else if $topStatsQuery?.data && $nodeStatusQuery?.data}
   231→  <Row>
   232→    <Col xs="12" lg="4" class="p-2">
   233→      {#key $nodeStatusQuery.data.jobsStatistics[0].histDuration}
   234→        <Histogram
   235→          data={convert2uplot(
   236→            $nodeStatusQuery.data.jobsStatistics[0].histDuration,
   237→          )}
   238→          title="Duration Distribution"
   239→          xlabel="Current Job Runtimes"
   240→          xunit="Runtime"
   241→          ylabel="Number of Jobs"
   242→          yunit="Jobs"
   243→          height="275"
   244→          usesBins
   245→          xtime
   246→          enableFlip
   247→        />
   248→      {/key}
   249→    </Col>
   250→    <Col xs="6" md="3" lg="2" class="p-2">
   251→

   252→
Top Users: Jobs

   253→        <Pie
   254→          {useAltColors}
   255→          canvasId="{canvasPrefix}-hpcpie-jobs-users"
   256→          size={colWidthJobs * 0.75}
   257→          sliceLabel="Jobs"
   258→          quantities={$topStatsQuery.data.topUserJobs.map(
   259→            (tu) => tu["totalJobs"],
   260→          )}
   261→          entities={$topStatsQuery.data.topUserJobs.map((tu) =>
   262→            scrambleNames ? scramble(tu.id) : tu.id,
   263→          )}
   264→        />
   265→

   266→    </Col>
   267→    <Col xs="6" md="3" lg="2" class="p-2">
   268→

   269→

   270→
   271→          User
   272→          Jobs
   273→

   274→        {#each $topStatsQuery.data.topUserJobs as tu, i}
   275→

   276→            <Icon name="circle-fill" style="color: {legendColors(i)};" />
   279→
   280→              {scrambleNames ? scramble(tu.id) : tu.id}
   286→
   287→
   288→            {#if tu?.name}
   289→              <Tooltip
   290→                target={`${canvasPrefix}-topName-jobs-${tu.id}`}
   291→                placement="left"
   292→                >{scrambleNames ? scramble(tu.name) : tu.name}</Tooltip
   293→              >
   294→            {/if}
   295→            {tu["totalJobs"]}
   296→

   297→        {/each}
   298→

   299→    </Col>
   300→
   301→    <Col xs="6" md="3" lg="2" class="p-2">
   302→
Top Projects: Jobs

   303→      <Pie
   304→        {useAltColors}
   305→        canvasId="{canvasPrefix}-hpcpie-jobs-projects"
   306→        size={colWidthJobs * 0.75}
   307→        sliceLabel={"Jobs"}
   308→        quantities={$topStatsQuery.data.topProjectJobs.map(
   309→          (tp) => tp["totalJobs"],
   310→        )}
   311→        entities={$topStatsQuery.data.topProjectJobs.map((tp) =>
   312→          scrambleNames ? scramble(tp.id) : tp.id,
   313→        )}
   314→      />
   315→    </Col>
   316→    <Col xs="6" md="3" lg="2" class="p-2">
   317→

   318→

   319→
   320→          Project
   321→          Jobs
   322→

   323→        {#each $topStatsQuery.data.topProjectJobs as tp, i}
   324→

   325→            <Icon name="circle-fill" style="color: {legendColors(i)};" />
   328→
   329→              {scrambleNames ? scramble(tp.id) : tp.id}
   335→
   336→
   337→            {tp["totalJobs"]}
   338→

   339→        {/each}
   340→

   341→    </Col>
   342→  </Row>
   343→{:else}
   344→  <Card class="mx-4" body color="warning"
   345→    >Cannot render job status charts: No data!</Card
   346→  >
   347→{/if}
   348→
   349→

   350→
   351→
   352→{#if $topStatsQuery?.fetching || $nodeStatusQuery?.fetching}
   353→  <Spinner />
   354→{:else if $topStatsQuery?.data && $nodeStatusQuery?.data}
   355→  <Row>
   356→    <Col xs="12" lg="4" class="p-2">
   357→      <Histogram
   358→        data={convert2uplot(
   359→          $nodeStatusQuery.data.jobsStatistics[0].histNumNodes,
   360→        )}
   361→        title="Number of Nodes Distribution"
   362→        xlabel="Allocated Nodes"
   363→        xunit="Nodes"
   364→        ylabel="Number of Jobs"
   365→        yunit="Jobs"
   366→        height="275"
   367→        enableFlip
   368→      />
   369→    </Col>
   370→    <Col xs="6" md="3" lg="2" class="p-2">
   371→

   372→
Top Users: Nodes

   373→        <Pie
   374→          {useAltColors}
   375→          canvasId="{canvasPrefix}-hpcpie-nodes-users"
   376→          size={colWidthNodes * 0.75}
   377→          sliceLabel="Nodes"
   378→          quantities={$topStatsQuery.data.topUserNodes.map(
   379→            (tu) => tu["totalNodes"],
   380→          )}
   381→          entities={$topStatsQuery.data.topUserNodes.map((tu) =>
   382→            scrambleNames ? scramble(tu.id) : tu.id,
   383→          )}
   384→        />
   385→

   386→    </Col>
   387→    <Col xs="6" md="3" lg="2" class="p-2">
   388→

   389→

   390→
   391→          User
   392→          Nodes
   393→

   394→        {#each $topStatsQuery.data.topUserNodes as tu, i}
   395→

   396→            <Icon name="circle-fill" style="color: {legendColors(i)};" />
   399→
   400→              {scrambleNames ? scramble(tu.id) : tu.id}
   406→
   407→
   408→            {#if tu?.name}
   409→              <Tooltip
   410→                target={`${canvasPrefix}-topName-nodes-${tu.id}`}
   411→                placement="left"
   412→                >{scrambleNames ? scramble(tu.name) : tu.name}</Tooltip
   413→              >
   414→            {/if}
   415→            {tu["totalNodes"]}
   416→

   417→        {/each}
   418→

   419→    </Col>
   420→
   421→    <Col xs="6" md="3" lg="2" class="p-2">
   422→
Top Projects: Nodes

   423→      <Pie
   424→        {useAltColors}
   425→        canvasId="{canvasPrefix}-hpcpie-nodes-projects"
   426→        size={colWidthNodes * 0.75}
   427→        sliceLabel={"Nodes"}
   428→        quantities={$topStatsQuery.data.topProjectNodes.map(
   429→          (tp) => tp["totalNodes"],
   430→        )}
   431→        entities={$topStatsQuery.data.topProjectNodes.map((tp) =>
   432→          scrambleNames ? scramble(tp.id) : tp.id,
   433→        )}
   434→      />
   435→    </Col>
   436→    <Col xs="6" md="3" lg="2" class="p-2">
   437→

   438→

   439→
   440→          Project
   441→          Nodes
   442→

   443→        {#each $topStatsQuery.data.topProjectNodes as tp, i}
   444→

   445→            <Icon name="circle-fill" style="color: {legendColors(i)};" />
   448→
   449→              {scrambleNames ? scramble(tp.id) : tp.id}
   455→
   456→
   457→            {tp["totalNodes"]}
   458→

   459→        {/each}
   460→

   461→    </Col>
   462→  </Row>
   463→{:else}
   464→  <Card class="mx-4" body color="warning"
   465→    >Cannot render node status charts: No data!</Card
   466→  >
   467→{/if}
   468→
   469→

   470→
   471→
   472→{#if $topStatsQuery?.fetching || $nodeStatusQuery?.fetching}
   473→  <Spinner />
   474→{:else if $topStatsQuery?.data && $nodeStatusQuery?.data}
   475→  <Row>
   476→    <Col xs="12" lg="4" class="p-2">
   477→      <Histogram
   478→        data={convert2uplot(
   479→          $nodeStatusQuery.data.jobsStatistics[0].histNumAccs,
   480→        )}
   481→        title="Number of Accelerators Distribution"
   482→        xlabel="Allocated Accs"
   483→        xunit="Accs"
   484→        ylabel="Number of Jobs"
   485→        yunit="Jobs"
   486→        height="275"
   487→        enableFlip
   488→      />
   489→    </Col>
   490→    <Col xs="6" md="3" lg="2" class="p-2">
   491→

   492→
Top Users: GPUs

   493→        <Pie
   494→          {useAltColors}
   495→          canvasId="{canvasPrefix}-hpcpie-accs-users"
   496→          size={colWidthAccs * 0.75}
   497→          sliceLabel="GPUs"
   498→          quantities={$topStatsQuery.data.topUserAccs.map(
   499→            (tu) => tu["totalAccs"],
   500→          )}
   501→          entities={$topStatsQuery.data.topUserAccs.map((tu) =>
   502→            scrambleNames ? scramble(tu.id) : tu.id,
   503→          )}
   504→        />
   505→

   506→    </Col>
   507→    <Col xs="6" md="3" lg="2" class="p-2">
   508→

   509→

   510→
   511→          User
   512→          GPUs
   513→

   514→        {#each $topStatsQuery.data.topUserAccs as tu, i}
   515→

   516→            <Icon name="circle-fill" style="color: {legendColors(i)};" />
   519→
   520→              {scrambleNames ? scramble(tu.id) : tu.id}
   526→
   527→
   528→            {#if tu?.name}
   529→              <Tooltip
   530→                target={`${canvasPrefix}-topName-accs-${tu.id}`}
   531→                placement="left"
   532→                >{scrambleNames ? scramble(tu.name) : tu.name}</Tooltip
   533→              >
   534→            {/if}
   535→            {tu["totalAccs"]}
   536→

   537→        {/each}
   538→

   539→    </Col>
   540→
   541→    <Col xs="6" md="3" lg="2" class="p-2">
   542→
Top Projects: GPUs

   543→      <Pie
   544→        {useAltColors}
   545→        canvasId="{canvasPrefix}-hpcpie-accs-projects"
   546→        size={colWidthAccs * 0.75}
   547→        sliceLabel={"GPUs"}
   548→        quantities={$topStatsQuery.data.topProjectAccs.map(
   549→          (tp) => tp["totalAccs"],
   550→        )}
   551→        entities={$topStatsQuery.data.topProjectAccs.map((tp) =>
   552→          scrambleNames ? scramble(tp.id) : tp.id,
   553→        )}
   554→      />
   555→    </Col>
   556→    <Col xs="6" md="3" lg="2" class="p-2">
   557→

   558→

   559→
   560→          Project
   561→          GPUs
   562→

   563→        {#each $topStatsQuery.data.topProjectAccs as tp, i}
   564→

   565→            <Icon name="circle-fill" style="color: {legendColors(i)};" />
   568→
   569→              {scrambleNames ? scramble(tp.id) : tp.id}
   575→
   576→
   577→            {tp["totalAccs"]}
   578→

   579→        {/each}
   580→

   581→    </Col>
   582→  </Row>
   583→{:else}
   584→  <Card class="mx-4" body color="warning"
   585→    >Cannot render accelerator status charts: No data!</Card
   586→  >
   587→{/if}
   588→

<Row class="justify-content-between">
  <Col class="mb-2 mb-md-0" xs="12" md="5" lg="4" xl="3">
    <InputGroup>
      <InputGroupText>
        <Icon name="bar-chart-line-fill" />
      </InputGroupText>
      <InputGroupText>Duration Bin Size</InputGroupText>
      <Input type="select" bind:value={numDurationBins}>
        {#each durationBinOptions as dbin}
          <option value={dbin}>{dbin}</option>
        {/each}
      </Input>
    </InputGroup>
  </Col>
  <Col xs="12" md="5" lg="4" xl="3">
    <Refresher
      initially={120}
      onRefresh={() => {
        pagingState = { page: 1, itemsPerPage: 10 };
        selectedHistograms = [...$state.snapshot(selectedHistograms)];
      }}
    />
  </Col>
</Row>

{#if $topStatsQuery?.fetching || $nodeStatusQuery?.fetching}
  <Spinner />
{:else if $topStatsQuery?.data && $nodeStatusQuery?.data}
  <Row>
    <Col xs="12" lg="4" class="p-2">
      {#key $nodeStatusQuery.data.jobsStatistics[0].histDuration}
        <Histogram
          data={convert2uplot(
            $nodeStatusQuery.data.jobsStatistics[0].histDuration,
          )}
          title="Duration Distribution"
          xlabel="Current Job Runtimes"
          xunit="Runtime"
          ylabel="Number of Jobs"
          yunit="Jobs"
          height="275"
          usesBins
          xtime
          enableFlip
        />
      {/key}
    </Col>
    <Col xs="6" md="3" lg="2" class="p-2">

Top Users: Jobs

        <Pie
          {useAltColors}
          canvasId="{canvasPrefix}-hpcpie-jobs-users"
          size={colWidthJobs * 0.75}
          sliceLabel="Jobs"
          quantities={$topStatsQuery.data.topUserJobs.map(
            (tu) => tu["totalJobs"],
          )}
          entities={$topStatsQuery.data.topUserJobs.map((tu) =>
            scrambleNames ? scramble(tu.id) : tu.id,
          )}
        />

    </Col>
    <Col xs="6" md="3" lg="2" class="p-2">

          User
          Jobs

        {#each $topStatsQuery.data.topUserJobs as tu, i}

            <Icon name="circle-fill" style="color: {legendColors(i)};" />

              {scrambleNames ? scramble(tu.id) : tu.id}

            {#if tu?.name}
              <Tooltip
                target={`${canvasPrefix}-topName-jobs-${tu.id}`}
                placement="left"
                >{scrambleNames ? scramble(tu.name) : tu.name}</Tooltip
              >
            {/if}
            {tu["totalJobs"]}

        {/each}

    </Col>

    <Col xs="6" md="3" lg="2" class="p-2">

Top Projects: Jobs

      <Pie
        {useAltColors}
        canvasId="{canvasPrefix}-hpcpie-jobs-projects"
        size={colWidthJobs * 0.75}
        sliceLabel={"Jobs"}
        quantities={$topStatsQuery.data.topProjectJobs.map(
          (tp) => tp["totalJobs"],
        )}
        entities={$topStatsQuery.data.topProjectJobs.map((tp) =>
          scrambleNames ? scramble(tp.id) : tp.id,
        )}
      />
    </Col>
    <Col xs="6" md="3" lg="2" class="p-2">

          Project
          Jobs

        {#each $topStatsQuery.data.topProjectJobs as tp, i}

            <Icon name="circle-fill" style="color: {legendColors(i)};" />

              {scrambleNames ? scramble(tp.id) : tp.id}

            {tp["totalJobs"]}

        {/each}

    </Col>
  </Row>
{:else}
  <Card class="mx-4" body color="warning"
    >Cannot render job status charts: No data!</Card
  >
{/if}

{#if $topStatsQuery?.fetching || $nodeStatusQuery?.fetching}
  <Spinner />
{:else if $topStatsQuery?.data && $nodeStatusQuery?.data}
  <Row>
    <Col xs="12" lg="4" class="p-2">
      <Histogram
        data={convert2uplot(
          $nodeStatusQuery.data.jobsStatistics[0].histNumNodes,
        )}
        title="Number of Nodes Distribution"
        xlabel="Allocated Nodes"
        xunit="Nodes"
        ylabel="Number of Jobs"
        yunit="Jobs"
        height="275"
        enableFlip
      />
    </Col>
    <Col xs="6" md="3" lg="2" class="p-2">

Top Users: Nodes

        <Pie
          {useAltColors}
          canvasId="{canvasPrefix}-hpcpie-nodes-users"
          size={colWidthNodes * 0.75}
          sliceLabel="Nodes"
          quantities={$topStatsQuery.data.topUserNodes.map(
            (tu) => tu["totalNodes"],
          )}
          entities={$topStatsQuery.data.topUserNodes.map((tu) =>
            scrambleNames ? scramble(tu.id) : tu.id,
          )}
        />

    </Col>
    <Col xs="6" md="3" lg="2" class="p-2">

          User
          Nodes

        {#each $topStatsQuery.data.topUserNodes as tu, i}

            <Icon name="circle-fill" style="color: {legendColors(i)};" />

              {scrambleNames ? scramble(tu.id) : tu.id}

            {#if tu?.name}
              <Tooltip
                target={`${canvasPrefix}-topName-nodes-${tu.id}`}
                placement="left"
                >{scrambleNames ? scramble(tu.name) : tu.name}</Tooltip
              >
            {/if}
            {tu["totalNodes"]}

        {/each}

    </Col>

    <Col xs="6" md="3" lg="2" class="p-2">

Top Projects: Nodes

      <Pie
        {useAltColors}
        canvasId="{canvasPrefix}-hpcpie-nodes-projects"
        size={colWidthNodes * 0.75}
        sliceLabel={"Nodes"}
        quantities={$topStatsQuery.data.topProjectNodes.map(
          (tp) => tp["totalNodes"],
        )}
        entities={$topStatsQuery.data.topProjectNodes.map((tp) =>
          scrambleNames ? scramble(tp.id) : tp.id,
        )}
      />
    </Col>
    <Col xs="6" md="3" lg="2" class="p-2">

          Project
          Nodes

        {#each $topStatsQuery.data.topProjectNodes as tp, i}

            <Icon name="circle-fill" style="color: {legendColors(i)};" />

              {scrambleNames ? scramble(tp.id) : tp.id}

            {tp["totalNodes"]}

        {/each}

    </Col>
  </Row>
{:else}
  <Card class="mx-4" body color="warning"
    >Cannot render node status charts: No data!</Card
  >
{/if}

{#if $topStatsQuery?.fetching || $nodeStatusQuery?.fetching}
  <Spinner />
{:else if $topStatsQuery?.data && $nodeStatusQuery?.data}
  <Row>
    <Col xs="12" lg="4" class="p-2">
      <Histogram
        data={convert2uplot(
          $nodeStatusQuery.data.jobsStatistics[0].histNumAccs,
        )}
        title="Number of Accelerators Distribution"
        xlabel="Allocated Accs"
        xunit="Accs"
        ylabel="Number of Jobs"
        yunit="Jobs"
        height="275"
        enableFlip
      />
    </Col>
    <Col xs="6" md="3" lg="2" class="p-2">

Top Users: GPUs

        <Pie
          {useAltColors}
          canvasId="{canvasPrefix}-hpcpie-accs-users"
          size={colWidthAccs * 0.75}
          sliceLabel="GPUs"
          quantities={$topStatsQuery.data.topUserAccs.map(
            (tu) => tu["totalAccs"],
          )}
          entities={$topStatsQuery.data.topUserAccs.map((tu) =>
            scrambleNames ? scramble(tu.id) : tu.id,
          )}
        />

    </Col>
    <Col xs="6" md="3" lg="2" class="p-2">

          User
          GPUs

        {#each $topStatsQuery.data.topUserAccs as tu, i}

            <Icon name="circle-fill" style="color: {legendColors(i)};" />

              {scrambleNames ? scramble(tu.id) : tu.id}

            {#if tu?.name}
              <Tooltip
                target={`${canvasPrefix}-topName-accs-${tu.id}`}
                placement="left"
                >{scrambleNames ? scramble(tu.name) : tu.name}</Tooltip
              >
            {/if}
            {tu["totalAccs"]}

        {/each}

    </Col>

    <Col xs="6" md="3" lg="2" class="p-2">

Top Projects: GPUs

      <Pie
        {useAltColors}
        canvasId="{canvasPrefix}-hpcpie-accs-projects"
        size={colWidthAccs * 0.75}
        sliceLabel={"GPUs"}
        quantities={$topStatsQuery.data.topProjectAccs.map(
          (tp) => tp["totalAccs"],
        )}
        entities={$topStatsQuery.data.topProjectAccs.map((tp) =>
          scrambleNames ? scramble(tp.id) : tp.id,
        )}
      />
    </Col>
    <Col xs="6" md="3" lg="2" class="p-2">

          Project
          GPUs

        {#each $topStatsQuery.data.topProjectAccs as tp, i}

            <Icon name="circle-fill" style="color: {legendColors(i)};" />

              {scrambleNames ? scramble(tp.id) : tp.id}

            {tp["totalAccs"]}

        {/each}

    </Col>
  </Row>
{:else}
  <Card class="mx-4" body color="warning"
    >Cannot render accelerator status charts: No data!</Card
  >
{/if}


     1→-- Migration 13: Add covering indexes for status/dashboard queries
     2→-- Column order: cluster (equality), job_state (equality), grouping col, then aggregated columns
     3→-- These indexes allow the status views to be served entirely from index scans.
     4→
     5→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user_stats
     6→  ON job (cluster, job_state, hpc_user, duration, start_time, num_nodes, num_hwthreads, num_acc);
     7→
     8→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project_stats
     9→  ON job (cluster, job_state, project, duration, start_time, num_nodes, num_hwthreads, num_acc);
    10→
    11→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_subcluster_stats
    12→  ON job (cluster, job_state, subcluster, duration, start_time, num_nodes, num_hwthreads, num_acc);
    13→
    14→-- Drop 3-col indexes that are now subsumed by the covering indexes above
    15→DROP INDEX IF EXISTS jobs_cluster_jobstate_user;
    16→DROP INDEX IF EXISTS jobs_cluster_jobstate_project;
    17→
    18→PRAGMA optimize;
    19→

-- Migration 13: Add covering indexes for status/dashboard queries
-- Column order: cluster (equality), job_state (equality), grouping col, then aggregated columns
-- These indexes allow the status views to be served entirely from index scans.

CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user_stats
  ON job (cluster, job_state, hpc_user, duration, start_time, num_nodes, num_hwthreads, num_acc);

CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project_stats
  ON job (cluster, job_state, project, duration, start_time, num_nodes, num_hwthreads, num_acc);

CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_subcluster_stats
  ON job (cluster, job_state, subcluster, duration, start_time, num_nodes, num_hwthreads, num_acc);

-- Drop 3-col indexes that are now subsumed by the covering indexes above
DROP INDEX IF EXISTS jobs_cluster_jobstate_user;
DROP INDEX IF EXISTS jobs_cluster_jobstate_project;

PRAGMA optimize;


Let me check a few more things before implementing.


Bash

grep -q 'errgroup' go.mod && echo "errgroup already in go.mod" || echo "need to add errgroup"

Check if errgroup is already a dependency


need to add errgroup


No files found


Found 2 files
internal/graph/schema.resolvers.go
internal/graph/util.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package graph
     7→
     8→import (
     9→	"context"
    10→	"fmt"
    11→	"math"
    12→	"slices"
    13→
    14→	"github.com/99designs/gqlgen/graphql"
    15→	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
    16→	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
    17→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    18→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    19→)
    20→
    21→const MAX_JOBS_FOR_ANALYSIS = 500
    22→
    23→// Helper function for the rooflineHeatmap GraphQL query placed here so that schema.resolvers.go is not too full.
    24→func (r *queryResolver) rooflineHeatmap(
    25→	ctx context.Context,
    26→	filter []*model.JobFilter,
    27→	rows int, cols int,
    28→	minX float64, minY float64, maxX float64, maxY float64,
    29→) ([][]float64, error) {
    30→	jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
    31→	if err != nil {
    32→		cclog.Error("Error while querying jobs for roofline")
    33→		return nil, err
    34→	}
    35→	if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
    36→		return nil, fmt.Errorf("GRAPH/UTIL > too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS)
    37→	}
    38→
    39→	fcols, frows := float64(cols), float64(rows)
    40→	minX, minY, maxX, maxY = math.Log10(minX), math.Log10(minY), math.Log10(maxX), math.Log10(maxY)
    41→	tiles := make([][]float64, rows)
    42→	for i := range tiles {
    43→		tiles[i] = make([]float64, cols)
    44→	}
    45→
    46→	for _, job := range jobs {
    47→		if job.MonitoringStatus == schema.MonitoringStatusDisabled || job.MonitoringStatus == schema.MonitoringStatusArchivingFailed {
    48→			continue
    49→		}
    50→
    51→		// metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
    52→		// resolution := 0
    53→
    54→		// for _, mc := range metricConfigs {
    55→		// 	resolution = max(resolution, mc.Timestep)
    56→		// }
    57→
    58→		jobdata, err := metricdispatch.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0)
    59→		if err != nil {
    60→			cclog.Warnf("Error while loading roofline metrics for job %d", *job.ID)
    61→			return nil, err
    62→		}
    63→
    64→		flops_, membw_ := jobdata["flops_any"], jobdata["mem_bw"]
    65→		if flops_ == nil && membw_ == nil {
    66→			cclog.Warnf("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", *job.ID)
    67→			continue
    68→			// return nil, fmt.Errorf("GRAPH/UTIL > 'flops_any' or 'mem_bw' missing for job %d", job.ID)
    69→		}
    70→
    71→		flops, ok1 := flops_["node"]
    72→		membw, ok2 := membw_["node"]
    73→		if !ok1 || !ok2 {
    74→			cclog.Info("rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
    75→			continue
    76→			// TODO/FIXME:
    77→			// return nil, errors.New("GRAPH/UTIL > todo: rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
    78→		}
    79→
    80→		for n := 0; n < len(flops.Series); n++ {
    81→			flopsSeries, membwSeries := flops.Series[n], membw.Series[n]
    82→			for i := 0; i < len(flopsSeries.Data); i++ {
    83→				if i >= len(membwSeries.Data) {
    84→					break
    85→				}
    86→
    87→				x, y := math.Log10(float64(flopsSeries.Data[i]/membwSeries.Data[i])), math.Log10(float64(flopsSeries.Data[i]))
    88→				if math.IsNaN(x) || math.IsNaN(y) || x < minX || x >= maxX || y < minY || y > maxY {
    89→					continue
    90→				}
    91→
    92→				x, y = math.Floor(((x-minX)/(maxX-minX))*fcols), math.Floor(((y-minY)/(maxY-minY))*frows)
    93→				if x < 0 || x >= fcols || y < 0 || y >= frows {
    94→					continue
    95→				}
    96→
    97→				tiles[int(y)][int(x)] += 1
    98→			}
    99→		}
   100→	}
   101→
   102→	return tiles, nil
   103→}
   104→
   105→// Helper function for the jobsFootprints GraphQL query placed here so that schema.resolvers.go is not too full.
   106→func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
   107→	jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
   108→	if err != nil {
   109→		cclog.Error("Error while querying jobs for footprint")
   110→		return nil, err
   111→	}
   112→	if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
   113→		return nil, fmt.Errorf("GRAPH/UTIL > too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS)
   114→	}
   115→
   116→	avgs := make([][]schema.Float, len(metrics))
   117→	for i := range avgs {
   118→		avgs[i] = make([]schema.Float, 0, len(jobs))
   119→	}
   120→
   121→	timeweights := new(model.TimeWeights)
   122→	timeweights.NodeHours = make([]schema.Float, 0, len(jobs))
   123→	timeweights.AccHours = make([]schema.Float, 0, len(jobs))
   124→	timeweights.CoreHours = make([]schema.Float, 0, len(jobs))
   125→
   126→	for _, job := range jobs {
   127→		if job.MonitoringStatus == schema.MonitoringStatusDisabled || job.MonitoringStatus == schema.MonitoringStatusArchivingFailed {
   128→			continue
   129→		}
   130→
   131→		if err := metricdispatch.LoadAverages(job, metrics, avgs, ctx); err != nil {
   132→			cclog.Error("Error while loading averages for footprint")
   133→			return nil, err
   134→		}
   135→
   136→		// #166 collect arrays: Null values or no null values?
   137→		timeweights.NodeHours = append(timeweights.NodeHours, schema.Float(float64(job.Duration)/60.0*float64(job.NumNodes)))
   138→		if job.NumAcc > 0 {
   139→			timeweights.AccHours = append(timeweights.AccHours, schema.Float(float64(job.Duration)/60.0*float64(job.NumAcc)))
   140→		} else {
   141→			timeweights.AccHours = append(timeweights.AccHours, schema.Float(1.0))
   142→		}
   143→		if job.NumHWThreads > 0 {
   144→			timeweights.CoreHours = append(timeweights.CoreHours, schema.Float(float64(job.Duration)/60.0*float64(job.NumHWThreads))) // SQLite HWThreads == Cores; numCoresForJob(job)
   145→		} else {
   146→			timeweights.CoreHours = append(timeweights.CoreHours, schema.Float(1.0))
   147→		}
   148→	}
   149→
   150→	res := make([]*model.MetricFootprints, len(avgs))
   151→	for i, arr := range avgs {
   152→		res[i] = &model.MetricFootprints{
   153→			Metric: metrics[i],
   154→			Data:   arr,
   155→		}
   156→	}
   157→
   158→	return &model.Footprints{
   159→		TimeWeights: timeweights,
   160→		Metrics:     res,
   161→	}, nil
   162→}
   163→
   164→// func numCoresForJob(job *schema.Job) (numCores int) {
   165→
   166→// 	subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster)
   167→// 	if scerr != nil {
   168→// 		return 1
   169→// 	}
   170→
   171→// 	totalJobCores := 0
   172→// 	topology := subcluster.Topology
   173→
   174→// 	for _, host := range job.Resources {
   175→// 		hwthreads := host.HWThreads
   176→// 		if hwthreads == nil {
   177→// 			hwthreads = topology.Node
   178→// 		}
   179→
   180→// 		hostCores, _ := topology.GetCoresFromHWThreads(hwthreads)
   181→// 		totalJobCores += len(hostCores)
   182→// 	}
   183→
   184→// 	return totalJobCores
   185→// }
   186→
   187→func requireField(ctx context.Context, name string) bool {
   188→	fields := graphql.CollectAllFields(ctx)
   189→
   190→	return slices.Contains(fields, name)
   191→}
   192→

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package graph

import (
	"context"
	"fmt"
	"math"
	"slices"

	"github.com/99designs/gqlgen/graphql"
	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
)

const MAX_JOBS_FOR_ANALYSIS = 500

// Helper function for the rooflineHeatmap GraphQL query placed here so that schema.resolvers.go is not too full.
func (r *queryResolver) rooflineHeatmap(
	ctx context.Context,
	filter []*model.JobFilter,
	rows int, cols int,
	minX float64, minY float64, maxX float64, maxY float64,
) ([][]float64, error) {
	jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
	if err != nil {
		cclog.Error("Error while querying jobs for roofline")
		return nil, err
	}
	if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
		return nil, fmt.Errorf("GRAPH/UTIL > too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS)
	}

	fcols, frows := float64(cols), float64(rows)
	minX, minY, maxX, maxY = math.Log10(minX), math.Log10(minY), math.Log10(maxX), math.Log10(maxY)
	tiles := make([][]float64, rows)
	for i := range tiles {
		tiles[i] = make([]float64, cols)
	}

	for _, job := range jobs {
		if job.MonitoringStatus == schema.MonitoringStatusDisabled || job.MonitoringStatus == schema.MonitoringStatusArchivingFailed {
			continue
		}

		// metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
		// resolution := 0

		// for _, mc := range metricConfigs {
		// 	resolution = max(resolution, mc.Timestep)
		// }

		jobdata, err := metricdispatch.LoadData(job, []string{"flops_any", "mem_bw"}, []schema.MetricScope{schema.MetricScopeNode}, ctx, 0)
		if err != nil {
			cclog.Warnf("Error while loading roofline metrics for job %d", *job.ID)
			return nil, err
		}

		flops_, membw_ := jobdata["flops_any"], jobdata["mem_bw"]
		if flops_ == nil && membw_ == nil {
			cclog.Warnf("rooflineHeatmap(): 'flops_any' or 'mem_bw' missing for job %d", *job.ID)
			continue
			// return nil, fmt.Errorf("GRAPH/UTIL > 'flops_any' or 'mem_bw' missing for job %d", job.ID)
		}

		flops, ok1 := flops_["node"]
		membw, ok2 := membw_["node"]
		if !ok1 || !ok2 {
			cclog.Info("rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
			continue
			// TODO/FIXME:
			// return nil, errors.New("GRAPH/UTIL > todo: rooflineHeatmap() query not implemented for where flops_any or mem_bw not available at 'node' level")
		}

		for n := 0; n < len(flops.Series); n++ {
			flopsSeries, membwSeries := flops.Series[n], membw.Series[n]
			for i := 0; i < len(flopsSeries.Data); i++ {
				if i >= len(membwSeries.Data) {
					break
				}

				x, y := math.Log10(float64(flopsSeries.Data[i]/membwSeries.Data[i])), math.Log10(float64(flopsSeries.Data[i]))
				if math.IsNaN(x) || math.IsNaN(y) || x < minX || x >= maxX || y < minY || y > maxY {
					continue
				}

				x, y = math.Floor(((x-minX)/(maxX-minX))*fcols), math.Floor(((y-minY)/(maxY-minY))*frows)
				if x < 0 || x >= fcols || y < 0 || y >= frows {
					continue
				}

				tiles[int(y)][int(x)] += 1
			}
		}
	}

	return tiles, nil
}

// Helper function for the jobsFootprints GraphQL query placed here so that schema.resolvers.go is not too full.
func (r *queryResolver) jobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) {
	jobs, err := r.Repo.QueryJobs(ctx, filter, &model.PageRequest{Page: 1, ItemsPerPage: MAX_JOBS_FOR_ANALYSIS + 1}, nil)
	if err != nil {
		cclog.Error("Error while querying jobs for footprint")
		return nil, err
	}
	if len(jobs) > MAX_JOBS_FOR_ANALYSIS {
		return nil, fmt.Errorf("GRAPH/UTIL > too many jobs matched (max: %d)", MAX_JOBS_FOR_ANALYSIS)
	}

	avgs := make([][]schema.Float, len(metrics))
	for i := range avgs {
		avgs[i] = make([]schema.Float, 0, len(jobs))
	}

	timeweights := new(model.TimeWeights)
	timeweights.NodeHours = make([]schema.Float, 0, len(jobs))
	timeweights.AccHours = make([]schema.Float, 0, len(jobs))
	timeweights.CoreHours = make([]schema.Float, 0, len(jobs))

	for _, job := range jobs {
		if job.MonitoringStatus == schema.MonitoringStatusDisabled || job.MonitoringStatus == schema.MonitoringStatusArchivingFailed {
			continue
		}

		if err := metricdispatch.LoadAverages(job, metrics, avgs, ctx); err != nil {
			cclog.Error("Error while loading averages for footprint")
			return nil, err
		}

		// #166 collect arrays: Null values or no null values?
		timeweights.NodeHours = append(timeweights.NodeHours, schema.Float(float64(job.Duration)/60.0*float64(job.NumNodes)))
		if job.NumAcc > 0 {
			timeweights.AccHours = append(timeweights.AccHours, schema.Float(float64(job.Duration)/60.0*float64(job.NumAcc)))
		} else {
			timeweights.AccHours = append(timeweights.AccHours, schema.Float(1.0))
		}
		if job.NumHWThreads > 0 {
			timeweights.CoreHours = append(timeweights.CoreHours, schema.Float(float64(job.Duration)/60.0*float64(job.NumHWThreads))) // SQLite HWThreads == Cores; numCoresForJob(job)
		} else {
			timeweights.CoreHours = append(timeweights.CoreHours, schema.Float(1.0))
		}
	}

	res := make([]*model.MetricFootprints, len(avgs))
	for i, arr := range avgs {
		res[i] = &model.MetricFootprints{
			Metric: metrics[i],
			Data:   arr,
		}
	}

	return &model.Footprints{
		TimeWeights: timeweights,
		Metrics:     res,
	}, nil
}

// func numCoresForJob(job *schema.Job) (numCores int) {

// 	subcluster, scerr := archive.GetSubCluster(job.Cluster, job.SubCluster)
// 	if scerr != nil {
// 		return 1
// 	}

// 	totalJobCores := 0
// 	topology := subcluster.Topology

// 	for _, host := range job.Resources {
// 		hwthreads := host.HWThreads
// 		if hwthreads == nil {
// 			hwthreads = topology.Node
// 		}

// 		hostCores, _ := topology.GetCoresFromHWThreads(hwthreads)
// 		totalJobCores += len(hostCores)
// 	}

// 	return totalJobCores
// }

func requireField(ctx context.Context, name string) bool {
	fields := graphql.CollectAllFields(ctx)

	return slices.Contains(fields, name)
}


     1→-- Reverse migration 13: Remove covering status indexes, restore 3-col indexes
     2→
     3→DROP INDEX IF EXISTS jobs_cluster_jobstate_user_stats;
     4→DROP INDEX IF EXISTS jobs_cluster_jobstate_project_stats;
     5→DROP INDEX IF EXISTS jobs_cluster_jobstate_subcluster_stats;
     6→
     7→-- Restore the original 3-col indexes
     8→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user
     9→  ON job (cluster, job_state, hpc_user);
    10→
    11→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project
    12→  ON job (cluster, job_state, project);
    13→

-- Reverse migration 13: Remove covering status indexes, restore 3-col indexes

DROP INDEX IF EXISTS jobs_cluster_jobstate_user_stats;
DROP INDEX IF EXISTS jobs_cluster_jobstate_project_stats;
DROP INDEX IF EXISTS jobs_cluster_jobstate_subcluster_stats;

-- Restore the original 3-col indexes
CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user
  ON job (cluster, job_state, hpc_user);

CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project
  ON job (cluster, job_state, project);


Now I have all the context. Let me implement all 4 changes. I'll work on them in parallel where possible.

**Change 1: Migration files + version bump + inline literal state value**


Write

-- Migration 14: Partial covering indexes for running jobs
-- Only running jobs are in the B-tree, so these indexes are tiny compared to
-- the full-table indexes from migration 13. SQLite uses them when the query
-- contains the literal `job_state = 'running'` (not a parameter placeholder).

CREATE INDEX IF NOT EXISTS jobs_running_user_stats
  ON job (cluster, hpc_user, num_nodes, num_hwthreads, num_acc, duration, start_time)
  WHERE job_state = 'running';

CREATE INDEX IF NOT EXISTS jobs_running_project_stats
  ON job (cluster, project, num_nodes, num_hwthreads, num_acc, duration, start_time)
  WHERE job_state = 'running';

CREATE INDEX IF NOT EXISTS jobs_running_subcluster_stats
  ON job (cluster, subcluster, num_nodes, num_hwthreads, num_acc, duration, start_time)
  WHERE job_state = 'running';

PRAGMA optimize;


File created successfully at: /Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/migrations/sqlite3/14_running-partial-indexes.up.sql

-- Migration 14: Partial covering indexes for running jobs
-- Only running jobs are in the B-tree, so these indexes are tiny compared to
-- the full-table indexes from migration 13. SQLite uses them when the query
-- contains the literal `job_state = 'running'` (not a parameter placeholder).

CREATE INDEX IF NOT EXISTS jobs_running_user_stats
  ON job (cluster, hpc_user, num_nodes, num_hwthreads, num_acc, duration, start_time)
  WHERE job_state = 'running';

CREATE INDEX IF NOT EXISTS jobs_running_project_stats
  ON job (cluster, project, num_nodes, num_hwthreads, num_acc, duration, start_time)
  WHERE job_state = 'running';

CREATE INDEX IF NOT EXISTS jobs_running_subcluster_stats
  ON job (cluster, subcluster, num_nodes, num_hwthreads, num_acc, duration, start_time)
  WHERE job_state = 'running';

PRAGMA optimize;


-- Reverse migration 14: Drop partial indexes for running jobs

DROP INDEX IF EXISTS jobs_running_user_stats;
DROP INDEX IF EXISTS jobs_running_project_stats;
DROP INDEX IF EXISTS jobs_running_subcluster_stats;


File created successfully at: /Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/migrations/sqlite3/14_running-partial-indexes.down.sql

-- Reverse migration 14: Drop partial indexes for running jobs

DROP INDEX IF EXISTS jobs_running_user_stats;
DROP INDEX IF EXISTS jobs_running_project_stats;
DROP INDEX IF EXISTS jobs_running_subcluster_stats;


Edit

// Version history:
//   - Version 13: Add covering indexes for status/dashboard queries (cluster, job_state, ...)
//   - Version 12: Add covering index for stats queries (cluster, start_time, hpc_user, ...)
//   - Version 11: Optimize job table indexes (reduce from ~78 to 48)
//   - Version 10: Node table
//
// Migration files are embedded at build time from the migrations directory.
const Version uint = 13

// Version history:
//   - Version 14: Partial covering indexes for running jobs (tiny B-tree vs millions)
//   - Version 13: Add covering indexes for status/dashboard queries (cluster, job_state, ...)
//   - Version 12: Add covering index for stats queries (cluster, start_time, hpc_user, ...)
//   - Version 11: Optimize job table indexes (reduce from ~78 to 48)
//   - Version 10: Node table
//
// Migration files are embedded at build time from the migrations directory.
const Version uint = 14


The file /Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/migration.go has been updated successfully.

// Version history:
//   - Version 14: Partial covering indexes for running jobs (tiny B-tree vs millions)
//   - Version 13: Add covering indexes for status/dashboard queries (cluster, job_state, ...)
//   - Version 12: Add covering index for stats queries (cluster, start_time, hpc_user, ...)
//   - Version 11: Optimize job table indexes (reduce from ~78 to 48)
//   - Version 10: Node table
//
// Migration files are embedded at build time from the migrations directory.
const Version uint = 14

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repository

import (
	"database/sql"
	"embed"
	"fmt"

	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/golang-migrate/migrate/v4"
	"github.com/golang-migrate/migrate/v4/database/sqlite3"
	"github.com/golang-migrate/migrate/v4/source/iofs"
)

// Version is the current database schema version required by this version of cc-backend.
// When the database schema changes, this version is incremented and a new migration file
// is added to internal/repository/migrations/sqlite3/.
//
// Version history:
//   - Version 13: Add covering indexes for status/dashboard queries (cluster, job_state, ...)
//   - Version 12: Add covering index for stats queries (cluster, start_time, hpc_user, ...)
//   - Version 11: Optimize job table indexes (reduce from ~78 to 48)
//   - Version 10: Node table
//
// Migration files are embedded at build time from the migrations directory.
const Version uint = 13

//go:embed migrations/*
var migrationFiles embed.FS

// checkDBVersion verifies that the database schema version matches the expected version.
// This is called automatically during Connect() to ensure schema compatibility.
//
// Returns an error if:
//   - Database version is older than expected (needs migration)
//   - Database version is newer than expected (needs app upgrade)
//   - Database is in a dirty state (failed migration)
//
// A "dirty" database indicates a migration was started but not completed successfully.
// This requires manual intervention to fix the database and force the version.
func checkDBVersion(db *sql.DB) error {
	driver, err := sqlite3.WithInstance(db, &sqlite3.Config{})
	if err != nil {
		return err
	}
	d, err := iofs.New(migrationFiles, "migrations/sqlite3")
	if err != nil {
		return err
	}

	m, err := migrate.NewWithInstance("iofs", d, "sqlite3", driver)
	if err != nil {
		return err
	}

	v, dirty, err := m.Version()
	if err != nil {
		if err == migrate.ErrNilVersion {
			cclog.Warn("Legacy database without version or missing database file!")
		} else {
			return err
		}
	}

	if v < Version {
		return fmt.Errorf("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version)
	} else if v > Version {
		return fmt.Errorf("unsupported database version %d, need %d.\nPlease refer to documentation how to downgrade db with external migrate tool", v, Version)
	}

	if dirty {
		return fmt.Errorf("last migration to version %d has failed, please fix the db manually and force version with -force-db flag", Version)
	}

	return nil
}

// getMigrateInstance creates a new migration instance for the given database file.
// This is used internally by MigrateDB, RevertDB, and ForceDB.
func getMigrateInstance(db string) (m *migrate.Migrate, err error) {
	d, err := iofs.New(migrationFiles, "migrations/sqlite3")
	if err != nil {
		return nil, err
	}

	m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("sqlite3://%s?_foreign_keys=on", db))
	if err != nil {
		return nil, err
	}

	return m, nil
}

// MigrateDB applies all pending database migrations to bring the schema up to date.
// This should be run with the -migrate-db flag before starting the application
// after upgrading to a new version that requires schema changes.
//
// Process:
//  1. Checks current database version
//  2. Applies all migrations from current version to target Version
//  3. Updates schema_migrations table to track applied migrations
//
// Important:
//   - Always backup your database before running migrations
//   - Migrations are irreversible without manual intervention
//   - If a migration fails, the database is marked "dirty" and requires manual fix
//
// Usage:
//
//	cc-backend -migrate-db
func MigrateDB(db string) error {
	m, err := getMigrateInstance(db)
	if err != nil {
		return err
	}

	v, dirty, err := m.Version()
	if err != nil {
		if err == migrate.ErrNilVersion {
			cclog.Info("Legacy database without version or missing database file!")
		} else {
			return err
		}
	}

	if v < Version {
		cclog.Infof("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version)
	}

	if dirty {
		return fmt.Errorf("last migration to version %d has failed, please fix the db manually and force version with -force-db flag", Version)
	}

	if err := m.Up(); err != nil {
		if err == migrate.ErrNoChange {
			cclog.Info("DB already up to date!")
		} else {
			return err
		}
	}

	m.Close()
	return nil
}

// RevertDB rolls back the database schema to the previous version (Version - 1).
// This is primarily used for testing or emergency rollback scenarios.
//
// Warning:
//   - This may cause data loss if newer schema added columns/tables
//   - Always backup before reverting
//   - Not all migrations are safely reversible
//
// Usage:
//
//	cc-backend -revert-db
func RevertDB(db string) error {
	m, err := getMigrateInstance(db)
	if err != nil {
		return err
	}

	if err := m.Migrate(Version - 1); err != nil {
		if err == migrate.ErrNoChange {
			cclog.Info("DB already up to date!")
		} else {
			return err
		}
	}

	m.Close()
	return nil
}

// ForceDB forces the database schema version to the current Version without running migrations.
// This is only used to recover from failed migrations that left the database in a "dirty" state.
//
// When to use:
//   - After manually fixing a failed migration
//   - When you've manually applied schema changes and need to update the version marker
//
// Warning:
//   - This does NOT apply any schema changes
//   - Only use after manually verifying the schema is correct
//   - Improper use can cause schema/version mismatch
//
// Usage:
//
//	cc-backend -force-db
func ForceDB(db string) error {
	m, err := getMigrateInstance(db)
	if err != nil {
		return err
	}

	if err := m.Force(int(Version)); err != nil {
		return err
	}

	m.Close()
	return nil
}

+//   - Version 14: Partial covering indexes for running jobs (tiny B-tree vs millions)


Now inline the literal state value in `BuildWhereClause` for single-state filters:


	if filter.State != nil {
		if len(filter.State) == 1 {
			singleStat := string(filter.State[0])
			query = query.Where("job.job_state = ?", singleStat)
		} else {

	if filter.State != nil {
		if len(filter.State) == 1 {
			// Inline literal value so SQLite can match partial indexes (WHERE job_state = 'running').
			// Safe: values come from validated GraphQL enum (model.JobState).
			singleStat := string(filter.State[0])
			query = query.Where(fmt.Sprintf("job.job_state = '%s'", singleStat))
		} else {


The file /Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/jobQuery.go has been updated successfully.

	if filter.State != nil {
		if len(filter.State) == 1 {
			// Inline literal value so SQLite can match partial indexes (WHERE job_state = 'running').
			// Safe: values come from validated GraphQL enum (model.JobState).
			singleStat := string(filter.State[0])
			query = query.Where(fmt.Sprintf("job.job_state = '%s'", singleStat))
		} else {

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package repository provides job query functionality with filtering, pagination,
// and security controls. This file contains the main query builders and security
// checks for job retrieval operations.
package repository

import (
	"context"
	"errors"
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/ClusterCockpit/cc-backend/internal/config"
	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
	"github.com/ClusterCockpit/cc-lib/v2/schema"
	sq "github.com/Masterminds/squirrel"
)

const (
	// Default initial capacity for job result slices
	defaultJobsCapacity = 50
)

// QueryJobs retrieves jobs from the database with optional filtering, pagination,
// and sorting. Security controls are automatically applied based on the user context.
//
// Parameters:
//   - ctx: Context containing user authentication information
//   - filters: Optional job filters (cluster, state, user, time ranges, etc.)
//   - page: Optional pagination parameters (page number and items per page)
//   - order: Optional sorting specification (column or footprint field)
//
// Returns a slice of jobs matching the criteria, or an error if the query fails.
// The function enforces role-based access control through SecurityCheck.
func (r *JobRepository) QueryJobs(
	ctx context.Context,
	filters []*model.JobFilter,
	page *model.PageRequest,
	order *model.OrderByInput,
) ([]*schema.Job, error) {
	query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job"))
	if qerr != nil {
		return nil, qerr
	}

	if order != nil {
		field := toSnakeCase(order.Field)
		if order.Type == "col" {
			switch order.Order {
			case model.SortDirectionEnumAsc:
				query = query.OrderBy(fmt.Sprintf("job.%s ASC", field))
			case model.SortDirectionEnumDesc:
				query = query.OrderBy(fmt.Sprintf("job.%s DESC", field))
			default:
				return nil, errors.New("invalid sorting order for column")
			}
		} else {
			// Order by footprint JSON field values
			query = query.Where("JSON_VALID(footprint)")
			switch order.Order {
			case model.SortDirectionEnumAsc:
				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") ASC", field))
			case model.SortDirectionEnumDesc:
				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") DESC", field))
			default:
				return nil, errors.New("invalid sorting order for footprint")
			}
		}
	}

	if page != nil && page.ItemsPerPage != -1 {
		limit := uint64(page.ItemsPerPage)
		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
	}

	for _, f := range filters {
		query = BuildWhereClause(f, query)
	}

	rows, err := query.RunWith(r.stmtCache).QueryContext(ctx)
	if err != nil {
		queryString, queryVars, _ := query.ToSql()
		return nil, fmt.Errorf("query failed [%s] %v: %w", queryString, queryVars, err)
	}
	defer rows.Close()

	jobs := make([]*schema.Job, 0, defaultJobsCapacity)
	for rows.Next() {
		job, err := scanJob(rows)
		if err != nil {
			cclog.Warnf("Error scanning job row: %v", err)
			return nil, fmt.Errorf("failed to scan job row: %w", err)
		}
		jobs = append(jobs, job)
	}

	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("error iterating job rows: %w", err)
	}

	return jobs, nil
}

// CountJobs returns the total number of jobs matching the given filters.
// Security controls are automatically applied based on the user context.
// Uses DISTINCT count to handle tag filters correctly (jobs may appear multiple
// times when joined with the tag table).
func (r *JobRepository) CountJobs(
	ctx context.Context,
	filters []*model.JobFilter,
) (int, error) {
	query, qerr := SecurityCheck(ctx, sq.Select("count(DISTINCT job.id)").From("job"))
	if qerr != nil {
		return 0, qerr
	}

	for _, f := range filters {
		query = BuildWhereClause(f, query)
	}

	var count int
	if err := query.RunWith(r.DB).QueryRowContext(ctx).Scan(&count); err != nil {
		return 0, fmt.Errorf("failed to count jobs: %w", err)
	}

	return count, nil
}

// SecurityCheckWithUser applies role-based access control filters to a job query
// based on the provided user's roles and permissions.
//
// Access rules by role:
//   - API role (exclusive): Full access to all jobs
//   - Admin/Support roles: Full access to all jobs
//   - Manager role: Access to jobs in managed projects plus own jobs
//   - User role: Access only to own jobs
//
// Returns an error if the user is nil or has no recognized roles.
func SecurityCheckWithUser(user *schema.User, query sq.SelectBuilder) (sq.SelectBuilder, error) {
	if user == nil {
		var qnil sq.SelectBuilder
		return qnil, fmt.Errorf("user context is nil")
	}

	switch {
	case len(user.Roles) == 1 && user.HasRole(schema.RoleAPI):
		return query, nil
	case user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport}):
		return query, nil
	case user.HasRole(schema.RoleManager):
		if len(user.Projects) != 0 {
			return query.Where(sq.Or{sq.Eq{"job.project": user.Projects}, sq.Eq{"job.hpc_user": user.Username}}), nil
		}
		cclog.Debugf("Manager '%s' has no assigned projects, restricting to personal jobs", user.Username)
		return query.Where("job.hpc_user = ?", user.Username), nil
	case user.HasRole(schema.RoleUser):
		return query.Where("job.hpc_user = ?", user.Username), nil
	default:
		var qnil sq.SelectBuilder
		return qnil, fmt.Errorf("user has no or unknown roles")
	}
}

// SecurityCheck extracts the user from the context and applies role-based access
// control filters to the query. This is a convenience wrapper around SecurityCheckWithUser.
func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) {
	user := GetUserFromContext(ctx)
	return SecurityCheckWithUser(user, query)
}

// BuildWhereClause constructs SQL WHERE conditions from a JobFilter and applies
// them to the query. Supports filtering by job properties (cluster, state, user),
// time ranges, resource usage, tags, and JSON field searches in meta_data,
// footprint, and resources columns.
func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.SelectBuilder {
	// Primary Key
	if filter.DbID != nil {
		dbIDs := make([]string, len(filter.DbID))
		copy(dbIDs, filter.DbID)
		query = query.Where(sq.Eq{"job.id": dbIDs})
	}
	// Explicit indices
	if filter.Cluster != nil {
		query = buildStringCondition("job.cluster", filter.Cluster, query)
	}
	if filter.SubCluster != nil {
		query = buildStringCondition("job.subcluster", filter.SubCluster, query)
	}
	if filter.Partition != nil {
		query = buildStringCondition("job.cluster_partition", filter.Partition, query)
	}
	if filter.State != nil {
		if len(filter.State) == 1 {
			singleStat := string(filter.State[0])
			query = query.Where("job.job_state = ?", singleStat)
		} else {
			states := make([]string, len(filter.State))
			for i, val := range filter.State {
				states[i] = string(val)
			}
			query = query.Where(sq.Eq{"job.job_state": states})
		}
	}
	if filter.Shared != nil {
		query = query.Where("job.shared = ?", *filter.Shared)
	}
	if filter.Project != nil {
		query = buildStringCondition("job.project", filter.Project, query)
	}
	if filter.User != nil {
		query = buildStringCondition("job.hpc_user", filter.User, query)
	}
	if filter.NumNodes != nil {
		query = buildIntCondition("job.num_nodes", filter.NumNodes, query)
	}
	if filter.NumAccelerators != nil {
		query = buildIntCondition("job.num_acc", filter.NumAccelerators, query)
	}
	if filter.NumHWThreads != nil {
		query = buildIntCondition("job.num_hwthreads", filter.NumHWThreads, query)
	}
	if filter.ArrayJobID != nil {
		query = query.Where("job.array_job_id = ?", *filter.ArrayJobID)
	}
	if filter.StartTime != nil {
		query = buildTimeCondition("job.start_time", filter.StartTime, query)
	}
	if filter.Duration != nil {
		query = buildIntCondition("job.duration", filter.Duration, query)
	}
	if filter.Energy != nil {
		query = buildFloatCondition("job.energy", filter.Energy, query)
	}
	// Indices on Tag Table
	if filter.Tags != nil {
		// This is an OR-Logic query: Returns all distinct jobs with at least one of the requested tags; TODO: AND-Logic query?
		query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags}).Distinct()
	}
	// No explicit Indices
	if filter.JobID != nil {
		query = buildStringCondition("job.job_id", filter.JobID, query)
	}
	// Queries Within JSONs
	if filter.MetricStats != nil {
		for _, ms := range filter.MetricStats {
			query = buildFloatJSONCondition(ms.MetricName, ms.Range, query)
		}
	}
	if filter.Node != nil {
		query = buildResourceJSONCondition("hostname", filter.Node, query)
	}
	if filter.JobName != nil {
		query = buildMetaJSONCondition("jobName", filter.JobName, query)
	}
	if filter.Schedule != nil {
		interactiveJobname := "interactive"
		switch *filter.Schedule {
		case "interactive":
			iFilter := model.StringInput{Eq: &interactiveJobname}
			query = buildMetaJSONCondition("jobName", &iFilter, query)
		case "batch":
			sFilter := model.StringInput{Neq: &interactiveJobname}
			query = buildMetaJSONCondition("jobName", &sFilter, query)
		}
	}

	// Configurable Filter to exclude recently started jobs, see config.go: ShortRunningJobsDuration
	if filter.MinRunningFor != nil {
		now := time.Now().Unix()
		// Only jobs whose start timestamp is more than MinRunningFor seconds in the past
		// If a job completed within the configured timeframe, it will still show up after the start_time matches the condition!
		query = query.Where(sq.Lt{"job.start_time": (now - int64(*filter.MinRunningFor))})
	}
	return query
}

// buildIntCondition creates clauses for integer range filters, using BETWEEN only if required.
func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuilder) sq.SelectBuilder {
	if cond.From != 1 && cond.To != 0 {
		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
	} else if cond.From != 1 && cond.To == 0 {
		return query.Where(field+" >= ?", cond.From)
	} else if cond.From == 1 && cond.To != 0 {
		return query.Where(field+" <= ?", cond.To)
	} else {
		return query
	}
}

// buildFloatCondition creates a clauses for float range filters, using BETWEEN only if required.
func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
	if cond.From != 1.0 && cond.To != 0.0 {
		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
	} else if cond.From != 1.0 && cond.To == 0.0 {
		return query.Where(field+" >= ?", cond.From)
	} else if cond.From == 1.0 && cond.To != 0.0 {
		return query.Where(field+" <= ?", cond.To)
	} else {
		return query
	}
}

// buildTimeCondition creates time range filters supporting absolute timestamps,
// relative time ranges (last6h, last24h, last7d, last30d), or open-ended ranges.
// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBuilder) sq.SelectBuilder {
	if cond.From != nil && cond.To != nil {
		return query.Where(field+" BETWEEN ? AND ?", cond.From.Unix(), cond.To.Unix())
	} else if cond.From != nil {
		return query.Where(field+" >= ?", cond.From.Unix())
	} else if cond.To != nil {
		return query.Where(field+" <= ?", cond.To.Unix())
	} else if cond.Range != "" {
		now := time.Now().Unix()
		var then int64
		switch cond.Range {
		case "last6h":
			then = now - (60 * 60 * 6)
		case "last24h":
			then = now - (60 * 60 * 24)
		case "last7d":
			then = now - (60 * 60 * 24 * 7)
		case "last30d":
			then = now - (60 * 60 * 24 * 30)
		default:
			cclog.Debugf("No known named timeRange: startTime.range = %s", cond.Range)
			return query
		}
		return query.Where(field+" >= ?", then)
	} else {
		return query
	}
}

// buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column, using BETWEEN only if required.
func buildFloatJSONCondition(jsonField string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
	query = query.Where("JSON_VALID(footprint)")
	if cond.From != 1.0 && cond.To != 0.0 {
		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") BETWEEN ? AND ?", cond.From, cond.To)
	} else if cond.From != 1.0 && cond.To == 0.0 {
		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") >= ?", cond.From)
	} else if cond.From == 1.0 && cond.To != 0.0 {
		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") <= ?", cond.To)
	} else {
		return query
	}
}

// buildStringCondition creates filters for string fields supporting equality,
// inequality, prefix, suffix, substring, and IN list matching.
func buildStringCondition(field string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder {
	if cond.Eq != nil {
		return query.Where(field+" = ?", *cond.Eq)
	}
	if cond.Neq != nil {
		return query.Where(field+" != ?", *cond.Neq)
	}
	if cond.StartsWith != nil {
		return query.Where(field+" LIKE ?", fmt.Sprint(*cond.StartsWith, "%"))
	}
	if cond.EndsWith != nil {
		return query.Where(field+" LIKE ?", fmt.Sprint("%", *cond.EndsWith))
	}
	if cond.Contains != nil {
		return query.Where(field+" LIKE ?", fmt.Sprint("%", *cond.Contains, "%"))
	}
	if cond.In != nil {
		queryElements := make([]string, len(cond.In))
		copy(queryElements, cond.In)
		return query.Where(sq.Or{sq.Eq{field: queryElements}})
	}
	return query
}

// buildMetaJSONCondition creates filters on fields within the meta_data JSON column.
func buildMetaJSONCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder {
	query = query.Where("JSON_VALID(meta_data)")
	if cond.Eq != nil {
		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") = ?", *cond.Eq)
	}
	if cond.Neq != nil {
		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") != ?", *cond.Neq)
	}
	if cond.StartsWith != nil {
		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint(*cond.StartsWith, "%"))
	}
	if cond.EndsWith != nil {
		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint("%", *cond.EndsWith))
	}
	if cond.Contains != nil {
		return query.Where("JSON_EXTRACT(meta_data, \"$."+jsonField+"\") LIKE ?", fmt.Sprint("%", *cond.Contains, "%"))
	}
	return query
}

// buildResourceJSONCondition creates filters on fields within the resources JSON array column.
// Uses json_each to search within array elements.
func buildResourceJSONCondition(jsonField string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder {
	query = query.Where("JSON_VALID(resources)")
	if cond.Eq != nil {
		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") = ?)", *cond.Eq)
	}
	if cond.Neq != nil { // Currently Unused
		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") != ?)", *cond.Neq)
	}
	if cond.StartsWith != nil { // Currently Unused
		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\")) LIKE ?)", fmt.Sprint(*cond.StartsWith, "%"))
	}
	if cond.EndsWith != nil { // Currently Unused
		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") LIKE ?)", fmt.Sprint("%", *cond.EndsWith))
	}
	if cond.Contains != nil {
		return query.Where("EXISTS (SELECT 1 FROM json_each(job.resources) WHERE json_extract(value, \"$."+jsonField+"\") LIKE ?)", fmt.Sprint("%", *cond.Contains, "%"))
	}
	return query
}

var (
	matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)")
	matchAllCap   = regexp.MustCompile("([a-z0-9])([A-Z])")
)

// toSnakeCase converts camelCase strings to snake_case for SQL column names.
// Includes security checks to prevent SQL injection attempts.
// Panics if potentially dangerous characters are detected.
func toSnakeCase(str string) string {
	for _, c := range str {
		if c == '\'' || c == '\\' || c == '"' || c == ';' || c == '-' || c == ' ' {
			cclog.Panicf("toSnakeCase: potentially dangerous character detected in input: %q", str)
		}
	}

	snake := matchFirstCap.ReplaceAllString(str, "${1}_${2}")
	snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}")
	return strings.ToLower(snake)
}

+      // Inline literal value so SQLite can match partial indexes (WHERE job_state = 'running').

+      // Safe: values come from validated GraphQL enum (model.JobState).

-      query = query.Where("job.job_state = ?", singleStat)

+      query = query.Where(fmt.Sprintf("job.job_state = '%s'", singleStat))


**Change 2: Request-scoped stats cache**


Write

// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
// All rights reserved. This file is part of cc-backend.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package graph

import (
	"context"
	"fmt"
	"sync"

	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
)

// statsGroupCache is a per-request cache for grouped JobsStatistics results.
// It deduplicates identical (filter+groupBy) SQL queries that arise when the
// frontend requests multiple sort/page slices of the same underlying data
// (e.g. topUserJobs, topUserNodes, topUserAccs all group by USER).
type statsGroupCache struct {
	mu      sync.Mutex
	entries map[string]*cacheEntry
}

type cacheEntry struct {
	once   sync.Once
	result []*model.JobsStatistics
	err    error
}

type ctxKey int

const statsGroupCacheKey ctxKey = iota

// newStatsGroupCache creates a new empty cache.
func newStatsGroupCache() *statsGroupCache {
	return &statsGroupCache{
		entries: make(map[string]*cacheEntry),
	}
}

// withStatsGroupCache injects a new cache into the context.
func withStatsGroupCache(ctx context.Context) context.Context {
	return context.WithValue(ctx, statsGroupCacheKey, newStatsGroupCache())
}

// getStatsGroupCache retrieves the cache from context, or nil if absent.
func getStatsGroupCache(ctx context.Context) *statsGroupCache {
	if c, ok := ctx.Value(statsGroupCacheKey).(*statsGroupCache); ok {
		return c
	}
	return nil
}

// cacheKey builds a deterministic string key from filter + groupBy.
func statsCacheKey(filter []*model.JobFilter, groupBy *model.Aggregate) string {
	return fmt.Sprintf("%v|%v", filter, *groupBy)
}

// getOrCompute returns cached results for the given key, computing them on
// first access via the provided function.
func (c *statsGroupCache) getOrCompute(
	key string,
	compute func() ([]*model.JobsStatistics, error),
) ([]*model.JobsStatistics, error) {
	c.mu.Lock()
	entry, ok := c.entries[key]
	if !ok {
		entry = &cacheEntry{}
		c.entries[key] = entry
	}
	c.mu.Unlock()

	entry.once.Do(func() {
		entry.result, entry.err = compute()
	})
	return entry.result, entry.err
}