     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package main provides the entry point for the ClusterCockpit backend server.
     7→// It orchestrates initialization of all subsystems including configuration,
     8→// database, authentication, and the HTTP server.
     9→package main
    10→
    11→import (
    12→	"context"
    13→	"encoding/json"
    14→	"fmt"
    15→	"os"
    16→	"os/signal"
    17→	"runtime/debug"
    18→	"strings"
    19→	"sync"
    20→	"syscall"
    21→	"time"
    22→
    23→	"github.com/ClusterCockpit/cc-backend/internal/archiver"
    24→	"github.com/ClusterCockpit/cc-backend/internal/auth"
    25→	"github.com/ClusterCockpit/cc-backend/internal/config"
    26→	"github.com/ClusterCockpit/cc-backend/internal/importer"
    27→	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
    28→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    29→	"github.com/ClusterCockpit/cc-backend/internal/tagger"
    30→	"github.com/ClusterCockpit/cc-backend/internal/taskmanager"
    31→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    32→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    33→	"github.com/ClusterCockpit/cc-backend/web"
    34→	ccconf "github.com/ClusterCockpit/cc-lib/v2/ccConfig"
    35→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    36→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    37→	"github.com/ClusterCockpit/cc-lib/v2/runtime"
    38→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    39→	"github.com/ClusterCockpit/cc-lib/v2/util"
    40→	"github.com/google/gops/agent"
    41→	"github.com/joho/godotenv"
    42→
    43→	_ "github.com/mattn/go-sqlite3"
    44→)
    45→
    46→const logoString = `
    47→ _____ _           _             ____           _          _ _
    48→/  ___| |_   _ ___| |_ ___ _ __ / ___|___   ___| | ___ __ (_) |_
    49→| |   | | | | / __| __/ _ \ '__| |   / _ \ / __| |/ / '_ \| | __|
    50→| |___| | |_| \__ \ ||  __/ |  | |__| (_) | (__|   <| |_) | | |_
    51→\_____|_|\__,_|___/\__\___|_|   \____\___/ \___|_|\_\ .__/|_|\__|
    52→                                                    |_|
    53→`
    54→
    55→// Environment variable names
    56→const (
    57→	envGOGC = "GOGC"
    58→)
    59→
    60→// Default configurations
    61→const (
    62→	defaultArchiveConfig = `{"kind":"file","path":"./var/job-archive"}`
    63→)
    64→
    65→var (
    66→	date    string
    67→	commit  string
    68→	version string
    69→)
    70→
    71→func printVersion() {
    72→	fmt.Print(logoString)
    73→	fmt.Printf("Version:\t%s\n", version)
    74→	fmt.Printf("Git hash:\t%s\n", commit)
    75→	fmt.Printf("Build time:\t%s\n", date)
    76→	fmt.Printf("SQL db version:\t%d\n", repository.Version)
    77→	fmt.Printf("Job archive version:\t%d\n", archive.Version)
    78→}
    79→
    80→func initGops() error {
    81→	if !flagGops {
    82→		return nil
    83→	}
    84→
    85→	if err := agent.Listen(agent.Options{}); err != nil {
    86→		return fmt.Errorf("starting gops agent: %w", err)
    87→	}
    88→	return nil
    89→}
    90→
    91→func loadEnvironment() error {
    92→	if err := godotenv.Load(); err != nil {
    93→		return fmt.Errorf("loading .env file: %w", err)
    94→	}
    95→	return nil
    96→}
    97→
    98→func initConfiguration() error {
    99→	ccconf.Init(flagConfigFile)
   100→
   101→	cfg := ccconf.GetPackageConfig("main")
   102→	if cfg == nil {
   103→		return fmt.Errorf("main configuration must be present")
   104→	}
   105→
   106→	config.Init(cfg)
   107→	return nil
   108→}
   109→
   110→func initDatabase() error {
   111→	if config.Keys.DbConfig != nil {
   112→		cfg := repository.DefaultConfig()
   113→		dc := config.Keys.DbConfig
   114→		if dc.CacheSizeMB > 0 {
   115→			cfg.DbCacheSizeMB = dc.CacheSizeMB
   116→		}
   117→		if dc.SoftHeapLimitMB > 0 {
   118→			cfg.DbSoftHeapLimitMB = dc.SoftHeapLimitMB
   119→		}
   120→		if dc.MaxOpenConnections > 0 {
   121→			cfg.MaxOpenConnections = dc.MaxOpenConnections
   122→		}
   123→		if dc.MaxIdleConnections > 0 {
   124→			cfg.MaxIdleConnections = dc.MaxIdleConnections
   125→		}
   126→		if dc.ConnectionMaxIdleTimeMins > 0 {
   127→			cfg.ConnectionMaxIdleTime = time.Duration(dc.ConnectionMaxIdleTimeMins) * time.Minute
   128→		}
   129→		repository.SetConfig(cfg)
   130→	}
   131→	repository.Connect(config.Keys.DB)
   132→	return nil
   133→}
   134→
   135→func handleDatabaseCommands() error {
   136→	if flagMigrateDB {
   137→		err := repository.MigrateDB(config.Keys.DB)
   138→		if err != nil {
   139→			return fmt.Errorf("migrating database to version %d: %w", repository.Version, err)
   140→		}
   141→		cclog.Exitf("MigrateDB Success: Migrated SQLite database at '%s' to version %d.\n",
   142→			config.Keys.DB, repository.Version)
   143→	}
   144→
   145→	if flagRevertDB {
   146→		err := repository.RevertDB(config.Keys.DB)
   147→		if err != nil {
   148→			return fmt.Errorf("reverting database to version %d: %w", repository.Version-1, err)
   149→		}
   150→		cclog.Exitf("RevertDB Success: Reverted SQLite database at '%s' to version %d.\n",


cmd/cc-backend/main.go:343:		metricstore.Init(mscfg, metrics, &wg)
cmd/cc-backend/main.go:373:	archiver.Start(repository.GetJobRepository(), ctx)
cmd/cc-backend/main.go:374:	taskmanager.Start(ccconf.GetPackageConfig("cron"), ccconf.GetPackageConfig("archive"))
cmd/cc-backend/main.go:406:			cclog.Info("Shutdown signal received")
cmd/cc-backend/main.go:411:		srv.Shutdown(ctx)
cmd/cc-backend/main.go:412:		util.FsWatcherShutdown()
cmd/cc-backend/main.go:413:		taskmanager.Shutdown()
cmd/cc-backend/server.go:407:		if err := s.server.Shutdown(shutdownCtx); err != nil {
cmd/cc-backend/server.go:418:func (s *Server) Shutdown(ctx context.Context) {
cmd/cc-backend/server.go:429:	if err := s.server.Shutdown(shutdownCtx); err != nil {
cmd/cc-backend/server.go:437:		metricstore.Shutdown()
cmd/cc-backend/server.go:440:	// Shutdown archiver with 10 second timeout for fast shutdown
cmd/cc-backend/server.go:441:	if err := archiver.Shutdown(10 * time.Second); err != nil {


   200→
   201→		job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), job.ID)
   202→		if err != nil {
   203→			handleError(err, http.StatusInternalServerError, rw)
   204→			return
   205→		}
   206→
   207→		if job.MonitoringStatus == schema.MonitoringStatusArchivingSuccessful {
   208→			job.Statistics, err = archive.GetStatistics(job)
   209→			if err != nil {
   210→				handleError(err, http.StatusInternalServerError, rw)
   211→				return
   212→			}
   213→		}
   214→
   215→		results = append(results, job)
   216→	}
   217→
   218→	cclog.Debugf("/api/jobs: %d jobs returned", len(results))
   219→	rw.Header().Add("Content-Type", "application/json")
   220→	bw := bufio.NewWriter(rw)
   221→	defer bw.Flush()
   222→
   223→	payload := GetJobsAPIResponse{
   224→		Jobs:  results,
   225→		Items: page.ItemsPerPage,
   226→		Page:  page.Page,
   227→	}
   228→
   229→	if err := json.NewEncoder(bw).Encode(payload); err != nil {
   230→		handleError(err, http.StatusInternalServerError, rw)
   231→		return
   232→	}
   233→}
   234→
   235→// getCompleteJobByID godoc
   236→// @summary   Get job meta and optional all metric data
   237→// @tags Job query
   238→// @description Job to get is specified by database ID
   239→// @description Returns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.
   240→// @produce     json
   241→// @param       id          path     int                  true "Database ID of Job"
   242→// @param       all-metrics query    bool                 false "Include all available metrics"
   243→// @success     200     {object} api.GetJobAPIResponse      "Job resource"
   244→// @failure     400     {object} api.ErrorResponse          "Bad Request"
   245→// @failure     401     {object} api.ErrorResponse          "Unauthorized"
   246→// @failure     403     {object} api.ErrorResponse          "Forbidden"
   247→// @failure     404     {object} api.ErrorResponse          "Resource not found"
   248→// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: finding job failed: sql: no rows in result set"
   249→// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
   250→// @security    ApiKeyAuth
   251→// @router      /api/jobs/{id} [get]
   252→func (api *RestAPI) getCompleteJobByID(rw http.ResponseWriter, r *http.Request) {
   253→	// Fetch job from db
   254→	id := chi.URLParam(r, "id")
   255→	var job *schema.Job
   256→	var err error
   257→	if id != "" {
   258→		id, e := strconv.ParseInt(id, 10, 64)
   259→		if e != nil {
   260→			handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw)
   261→			return
   262→		}
   263→
   264→		job, err = api.JobRepository.FindByID(r.Context(), id) // Get Job from Repo by ID
   265→	} else {
   266→		handleError(fmt.Errorf("the parameter 'id' is required"), http.StatusBadRequest, rw)
   267→		return
   268→	}
   269→	if err != nil {
   270→		handleError(fmt.Errorf("finding job with db id %s failed: %w", id, err), http.StatusUnprocessableEntity, rw)
   271→		return
   272→	}
   273→
   274→	job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), job.ID)
   275→	if err != nil {
   276→		handleError(err, http.StatusInternalServerError, rw)
   277→		return
   278→
   279→	}
   280→	if _, err = api.JobRepository.FetchMetadata(job); err != nil {
   281→
   282→		handleError(err, http.StatusInternalServerError, rw)
   283→		return
   284→	}
   285→
   286→	var scopes []schema.MetricScope
   287→
   288→	if job.NumNodes == 1 {
   289→		scopes = []schema.MetricScope{"core"}
   290→	} else {
   291→		scopes = []schema.MetricScope{"node"}
   292→	}
   293→
   294→	var data schema.JobData
   295→
   296→	metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
   297→	resolution := 0
   298→
   299→	for _, mc := range metricConfigs {
   300→		resolution = max(resolution, mc.Timestep)
   301→	}
   302→
   303→	if r.URL.Query().Get("all-metrics") == "true" {
   304→		data, err = metricdispatch.LoadData(job, nil, scopes, r.Context(), resolution)
   305→		if err != nil {
   306→			cclog.Warnf("REST: error while loading all-metrics job data for JobID %d on %s", job.JobID, job.Cluster)
   307→			return
   308→		}
   309→	}
   310→
   311→	cclog.Debugf("/api/job/%s: get job %d", id, job.JobID)
   312→	rw.Header().Add("Content-Type", "application/json")
   313→	bw := bufio.NewWriter(rw)
   314→	defer bw.Flush()
   315→
   316→	payload := GetCompleteJobAPIResponse{
   317→		Meta: job,
   318→		Data: data,
   319→	}
   320→
   321→	if err := json.NewEncoder(bw).Encode(payload); err != nil {
   322→		handleError(err, http.StatusInternalServerError, rw)
   323→		return
   324→	}
   325→}
   326→
   327→// getJobByID godoc
   328→// @summary   Get job meta and configurable metric data
   329→// @tags Job query
   330→// @description Job to get is specified by database ID
   331→// @description Returns full job resource information according to 'Job' scheme and all metrics according to 'JobData'.
   332→// @accept      json
   333→// @produce     json
   334→// @param       id          path     int                  true "Database ID of Job"
   335→// @param       request     body     api.GetJobAPIRequest true  "Array of metric names"
   336→// @success     200     {object} api.GetJobAPIResponse      "Job resource"
   337→// @failure     400     {object} api.ErrorResponse          "Bad Request"
   338→// @failure     401     {object} api.ErrorResponse          "Unauthorized"
   339→// @failure     403     {object} api.ErrorResponse          "Forbidden"
   340→// @failure     404     {object} api.ErrorResponse          "Resource not found"
   341→// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: finding job failed: sql: no rows in result set"
   342→// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
   343→// @security    ApiKeyAuth
   344→// @router      /api/jobs/{id} [post]
   345→func (api *RestAPI) getJobByID(rw http.ResponseWriter, r *http.Request) {
   346→	// Fetch job from db
   347→	id := chi.URLParam(r, "id")
   348→	var job *schema.Job
   349→	var err error
   350→	if id != "" {
   351→		id, e := strconv.ParseInt(id, 10, 64)
   352→		if e != nil {
   353→			handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw)
   354→			return
   355→		}
   356→
   357→		job, err = api.JobRepository.FindByID(r.Context(), id)
   358→	} else {
   359→		handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw)
   360→		return
   361→	}
   362→	if err != nil {
   363→		handleError(fmt.Errorf("finding job with db id %s failed: %w", id, err), http.StatusUnprocessableEntity, rw)
   364→		return
   365→	}
   366→
   367→	job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), job.ID)
   368→	if err != nil {
   369→		handleError(err, http.StatusInternalServerError, rw)
   370→		return
   371→
   372→	}
   373→	if _, err = api.JobRepository.FetchMetadata(job); err != nil {
   374→
   375→		handleError(err, http.StatusInternalServerError, rw)
   376→		return
   377→	}
   378→
   379→	var metrics GetJobAPIRequest
   380→	if err = decode(r.Body, &metrics); err != nil {
   381→		handleError(fmt.Errorf("decoding request failed: %w", err), http.StatusBadRequest, rw)
   382→		return
   383→	}
   384→
   385→	var scopes []schema.MetricScope
   386→
   387→	if job.NumNodes == 1 {
   388→		scopes = []schema.MetricScope{"core"}
   389→	} else {
   390→		scopes = []schema.MetricScope{"node"}
   391→	}
   392→
   393→	metricConfigs := archive.GetCluster(job.Cluster).MetricConfig
   394→	resolution := 0
   395→
   396→	for _, mc := range metricConfigs {
   397→		resolution = max(resolution, mc.Timestep)
   398→	}
   399→
   400→	data, err := metricdispatch.LoadData(job, metrics, scopes, r.Context(), resolution)
   401→	if err != nil {
   402→		cclog.Warnf("REST: error while loading job data for JobID %d on %s", job.JobID, job.Cluster)
   403→		return
   404→	}
   405→
   406→	res := []*JobMetricWithName{}
   407→	for name, md := range data {
   408→		for scope, metric := range md {
   409→			res = append(res, &JobMetricWithName{
   410→				Name:   name,
   411→				Scope:  scope,
   412→				Metric: metric,
   413→			})
   414→		}
   415→	}
   416→
   417→	cclog.Debugf("/api/job/%s: get job %d", id, job.JobID)
   418→	rw.Header().Add("Content-Type", "application/json")
   419→	bw := bufio.NewWriter(rw)
   420→	defer bw.Flush()
   421→
   422→	payload := GetJobAPIResponse{
   423→		Meta: job,
   424→		Data: res,
   425→	}
   426→
   427→	if err := json.NewEncoder(bw).Encode(payload); err != nil {
   428→		handleError(err, http.StatusInternalServerError, rw)
   429→		return
   430→	}
   431→}
   432→
   433→// editMeta godoc
   434→// @summary    Edit meta-data json of job identified by database id
   435→// @tags Job add and modify
   436→// @description Edit key value pairs in job metadata json of job specified by database id
   437→// @description If a key already exists its content will be overwritten
   438→// @accept      json
   439→// @produce     json
   440→// @param       id      path     int                  true "Job Database ID"
   441→// @param       request body     api.EditMetaRequest  true "Metadata Key value pair to add or update"
   442→// @success     200     {object} schema.Job                "Updated job resource"
   443→// @failure     400     {object} api.ErrorResponse         "Bad Request"
   444→// @failure     401     {object} api.ErrorResponse         "Unauthorized"
   445→// @failure     404     {object} api.ErrorResponse         "Job does not exist"
   446→// @failure     500     {object} api.ErrorResponse         "Internal Server Error"
   447→// @security    ApiKeyAuth
   448→// @router      /api/jobs/edit_meta/{id} [patch]
   449→func (api *RestAPI) editMeta(rw http.ResponseWriter, r *http.Request) {
   450→	id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
   451→	if err != nil {
   452→		handleError(fmt.Errorf("parsing job ID failed: %w", err), http.StatusBadRequest, rw)
   453→		return
   454→	}
   455→
   456→	job, err := api.JobRepository.FindByID(r.Context(), id)
   457→	if err != nil {
   458→		handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw)
   459→		return
   460→	}
   461→
   462→	var req EditMetaRequest
   463→	if err := decode(r.Body, &req); err != nil {
   464→		handleError(fmt.Errorf("decoding request failed: %w", err), http.StatusBadRequest, rw)
   465→		return
   466→	}
   467→
   468→	if err := api.JobRepository.UpdateMetadata(job, req.Key, req.Value); err != nil {
   469→		handleError(fmt.Errorf("updating metadata failed: %w", err), http.StatusInternalServerError, rw)
   470→		return
   471→	}
   472→
   473→	rw.Header().Add("Content-Type", "application/json")
   474→	rw.WriteHeader(http.StatusOK)
   475→	if err := json.NewEncoder(rw).Encode(job); err != nil {
   476→		cclog.Errorf("Failed to encode job response: %v", err)
   477→	}
   478→}
   479→
   480→// editMetaByRequest godoc
   481→// @summary    Edit meta-data json of job identified by request
   482→// @tags Job add and modify
   483→// @description Edit key value pairs in metadata json of job specified by jobID, StartTime and Cluster
   484→// @description If a key already exists its content will be overwritten
   485→// @accept      json
   486→// @produce     json
   487→// @param       request body     api.JobMetaRequest   true "Specifies job and payload to add or update"
   488→// @success     200     {object} schema.Job                "Updated job resource"
   489→// @failure     400     {object} api.ErrorResponse         "Bad Request"
   490→// @failure     401     {object} api.ErrorResponse         "Unauthorized"
   491→// @failure     404     {object} api.ErrorResponse         "Job does not exist"
   492→// @failure     500     {object} api.ErrorResponse         "Internal Server Error"
   493→// @security    ApiKeyAuth
   494→// @router      /api/jobs/edit_meta/ [patch]
   495→func (api *RestAPI) editMetaByRequest(rw http.ResponseWriter, r *http.Request) {
   496→	// Parse request body
   497→	req := JobMetaRequest{}
   498→	if err := decode(r.Body, &req); err != nil {
   499→		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)


<persisted-output>
Output too large (59.9KB). Full output saved to: /Users/jan/.claude/projects/-Users-jan-prg-ClusterCockpit-cc-backend/3a41c7a4-abf8-484b-96ae-ee5618eea5ba/tool-results/REDACTED.txt

Preview (first 2KB):
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-182-func (r *NodeRepository) AddNode(node *schema.NodeDB) (int64, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-183-	var err error
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-184-
internal/repository/node.go:185:	res, err := r.DB.NamedExec(NamedNodeInsert, node)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-186-	if err != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-187-		cclog.Errorf("Error while adding node '%v' to database", node.Hostname)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-188-		return 0, err
--
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-235-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-236-	nodeState.NodeID = id
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-237-
internal/repository/node.go:238:	_, err := r.DB.NamedExec(NamedNodeStateInsert, nodeState)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-239-	if err != nil {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-240-		cclog.Errorf("Error while adding node state for '%v' to database", hostname)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-241-		return err
--
internal/repository/node.go-291-				node := schema.NodeDB{
internal/repository/node.go-292-					Hostname: u.Hostname, Cluster: u.Cluster, SubCluster: subcluster,
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-293-				}
internal/repository/node.go:294:				res, insertErr := stmtInsertNode.Exec(&node)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/node.go-295-				if insertErr != nil {
internal/repository/node.go-296-					cclog.Errorf("BatchUpdateNodeStates: insert node '%s': %v", u.Hostname, insertErr)
...
</persisted-output>


   545→func (api *RestAPI) tagJob(rw http.ResponseWriter, r *http.Request) {
   546→	id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
   547→	if err != nil {
   548→		handleError(fmt.Errorf("parsing job ID failed: %w", err), http.StatusBadRequest, rw)
   549→		return
   550→	}
   551→
   552→	job, err := api.JobRepository.FindByID(r.Context(), id)
   553→	if err != nil {
   554→		handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw)
   555→		return
   556→	}
   557→
   558→	job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), job.ID)
   559→	if err != nil {
   560→		handleError(fmt.Errorf("getting tags failed: %w", err), http.StatusInternalServerError, rw)
   561→		return
   562→	}
   563→
   564→	var req TagJobAPIRequest
   565→	if err := decode(r.Body, &req); err != nil {
   566→		handleError(fmt.Errorf("decoding request failed: %w", err), http.StatusBadRequest, rw)
   567→		return
   568→	}
   569→
   570→	for _, tag := range req {
   571→		tagID, err := api.JobRepository.AddTagOrCreate(repository.GetUserFromContext(r.Context()), *job.ID, tag.Type, tag.Name, tag.Scope)
   572→		if err != nil {
   573→			handleError(fmt.Errorf("adding tag failed: %w", err), http.StatusInternalServerError, rw)
   574→			return
   575→		}
   576→
   577→		job.Tags = append(job.Tags, &schema.Tag{
   578→			ID:    tagID,
   579→			Type:  tag.Type,
   580→			Name:  tag.Name,
   581→			Scope: tag.Scope,
   582→		})
   583→	}
   584→
   585→	rw.Header().Add("Content-Type", "application/json")
   586→	rw.WriteHeader(http.StatusOK)
   587→	if err := json.NewEncoder(rw).Encode(job); err != nil {
   588→		cclog.Errorf("Failed to encode job response: %v", err)
   589→	}
   590→}
   591→
   592→// removeTagJob godoc
   593→// @summary     Removes one or more tags from a job
   594→// @tags Job add and modify
   595→// @description Removes tag(s) from a job specified by DB ID. Name and Type of Tag(s) must match.
   596→// @description Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API.
   597→// @description If tagged job is already finished: Tag will be removed from respective archive files.
   598→// @accept      json
   599→// @produce     json
   600→// @param       id      path     int                  true "Job Database ID"
   601→// @param       request body     api.TagJobAPIRequest true "Array of tag-objects to remove"
   602→// @success     200     {object} schema.Job                "Updated job resource"
   603→// @failure     400     {object} api.ErrorResponse         "Bad Request"
   604→// @failure     401     {object} api.ErrorResponse         "Unauthorized"
   605→// @failure     404     {object} api.ErrorResponse         "Job or tag does not exist"
   606→// @failure     500     {object} api.ErrorResponse         "Internal Server Error"
   607→// @security    ApiKeyAuth
   608→// @router      /jobs/tag_job/{id} [delete]
   609→func (api *RestAPI) removeTagJob(rw http.ResponseWriter, r *http.Request) {
   610→	id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
   611→	if err != nil {
   612→		handleError(fmt.Errorf("parsing job ID failed: %w", err), http.StatusBadRequest, rw)
   613→		return
   614→	}
   615→
   616→	job, err := api.JobRepository.FindByID(r.Context(), id)
   617→	if err != nil {
   618→		handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw)
   619→		return
   620→	}
   621→
   622→	job.Tags, err = api.JobRepository.GetTags(repository.GetUserFromContext(r.Context()), job.ID)
   623→	if err != nil {
   624→		handleError(fmt.Errorf("getting tags failed: %w", err), http.StatusInternalServerError, rw)
   625→		return
   626→	}
   627→
   628→	var req TagJobAPIRequest
   629→	if err := decode(r.Body, &req); err != nil {
   630→		handleError(fmt.Errorf("decoding request failed: %w", err), http.StatusBadRequest, rw)
   631→		return
   632→	}
   633→
   634→	for _, rtag := range req {
   635→		// Only Global and Admin Tags
   636→		if rtag.Scope != "global" && rtag.Scope != "admin" {
   637→			cclog.Warnf("Cannot delete private tag for job %d: Skip", job.JobID)
   638→			continue
   639→		}
   640→
   641→		remainingTags, err := api.JobRepository.RemoveJobTagByRequest(repository.GetUserFromContext(r.Context()), *job.ID, rtag.Type, rtag.Name, rtag.Scope)
   642→		if err != nil {
   643→			handleError(fmt.Errorf("removing tag failed: %w", err), http.StatusInternalServerError, rw)
   644→			return
   645→		}
   646→
   647→		job.Tags = remainingTags
   648→	}
   649→
   650→	rw.Header().Add("Content-Type", "application/json")
   651→	rw.WriteHeader(http.StatusOK)
   652→	if err := json.NewEncoder(rw).Encode(job); err != nil {
   653→		cclog.Errorf("Failed to encode job response: %v", err)
   654→	}
   655→}
   656→
   657→// removeTags godoc
   658→// @summary     Removes all tags and job-relations for type:name tuple
   659→// @tags Tag remove
   660→// @description Removes tags by type and name. Name and Type of Tag(s) must match.
   661→// @description Tag Scope is required for matching, options: "global", "admin". Private tags can not be deleted via API.
   662→// @description Tag wills be removed from respective archive files.
   663→// @accept      json
   664→// @produce     plain
   665→// @param       request body     api.TagJobAPIRequest true "Array of tag-objects to remove"
   666→// @success     200     {string} string                    "Success Response"
   667→// @failure     400     {object} api.ErrorResponse         "Bad Request"
   668→// @failure     401     {object} api.ErrorResponse         "Unauthorized"
   669→// @failure     404     {object} api.ErrorResponse         "Job or tag does not exist"
   670→// @failure     500     {object} api.ErrorResponse         "Internal Server Error"
   671→// @security    ApiKeyAuth
   672→// @router      /tags/ [delete]
   673→func (api *RestAPI) removeTags(rw http.ResponseWriter, r *http.Request) {
   674→	var req TagJobAPIRequest
   675→	if err := decode(r.Body, &req); err != nil {
   676→		handleError(fmt.Errorf("decoding request failed: %w", err), http.StatusBadRequest, rw)
   677→		return
   678→	}
   679→
   680→	targetCount := len(req)
   681→	currentCount := 0
   682→	for _, rtag := range req {
   683→		// Only Global and Admin Tags
   684→		if rtag.Scope != "global" && rtag.Scope != "admin" {
   685→			cclog.Warn("Cannot delete private tag: Skip")
   686→			continue
   687→		}
   688→
   689→		err := api.JobRepository.RemoveTagByRequest(rtag.Type, rtag.Name, rtag.Scope)
   690→		if err != nil {
   691→			handleError(fmt.Errorf("removing tag failed: %w", err), http.StatusInternalServerError, rw)
   692→			return
   693→		}
   694→		currentCount++


   717→func (api *RestAPI) startJob(rw http.ResponseWriter, r *http.Request) {
   718→	req := schema.Job{
   719→		Shared:           "none",
   720→		MonitoringStatus: schema.MonitoringStatusRunningOrArchiving,
   721→	}
   722→	if err := decode(r.Body, &req); err != nil {
   723→		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
   724→		return
   725→	}
   726→
   727→	cclog.Debugf("REST: %s", req.GoString())
   728→	req.State = schema.JobStateRunning
   729→
   730→	if err := importer.SanityChecks(&req); err != nil {
   731→		handleError(err, http.StatusBadRequest, rw)
   732→		return
   733→	}
   734→
   735→	// aquire lock to avoid race condition between API calls
   736→	var unlockOnce sync.Once
   737→	api.RepositoryMutex.Lock()
   738→	defer unlockOnce.Do(api.RepositoryMutex.Unlock)
   739→
   740→	// Check if combination of (job_id, cluster_id, start_time) already exists:
   741→	jobs, err := api.JobRepository.FindAll(&req.JobID, &req.Cluster, nil)
   742→	if err != nil && err != sql.ErrNoRows {
   743→		handleError(fmt.Errorf("checking for duplicate failed: %w", err), http.StatusInternalServerError, rw)
   744→		return
   745→	}
   746→	if err == nil {
   747→		for _, job := range jobs {
   748→			// Check if jobs are within the same day (prevent duplicates)
   749→			if (req.StartTime - job.StartTime) < secondsPerDay {
   750→				handleError(fmt.Errorf("a job with that jobId, cluster and startTime already exists: dbid: %d, jobid: %d", *job.ID, job.JobID), http.StatusUnprocessableEntity, rw)
   751→				return
   752→			}
   753→		}
   754→	}
   755→
   756→	// When tags are present, insert directly into the job table so that the
   757→	// returned ID can be used with AddTagOrCreate (which queries the job table).
   758→	// Jobs without tags use the cache path as before.
   759→	var id int64
   760→	if len(req.Tags) > 0 {
   761→		id, err = api.JobRepository.StartDirect(&req)
   762→	} else {
   763→		id, err = api.JobRepository.Start(&req)
   764→	}
   765→	if err != nil {
   766→		handleError(fmt.Errorf("insert into database failed: %w", err), http.StatusInternalServerError, rw)
   767→		return
   768→	}
   769→	// unlock here, adding Tags can be async
   770→	unlockOnce.Do(api.RepositoryMutex.Unlock)
   771→
   772→	for _, tag := range req.Tags {
   773→		if _, err := api.JobRepository.AddTagOrCreate(repository.GetUserFromContext(r.Context()), id, tag.Type, tag.Name, tag.Scope); err != nil {
   774→			handleError(fmt.Errorf("adding tag to new job %d failed: %w", id, err), http.StatusInternalServerError, rw)
   775→			return
   776→		}
   777→	}
   778→
   779→	cclog.Infof("new job (id: %d): cluster=%s, jobId=%d, user=%s, startTime=%d", id, req.Cluster, req.JobID, req.User, req.StartTime)
   780→	rw.Header().Add("Content-Type", "application/json")
   781→	rw.WriteHeader(http.StatusCreated)
   782→	if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{
   783→		Message: "success",
   784→	}); err != nil {
   785→		cclog.Errorf("Failed to encode response: %v", err)
   786→	}
   787→}
   788→
   789→// stopJobByRequest godoc
   790→// @summary     Marks job as completed and triggers archiving
   791→// @tags Job add and modify
   792→// @description Job to stop is specified by request body. All fields are required in this case.
   793→// @description Returns full job resource information according to 'Job' scheme.
   794→// @produce     json
   795→// @param       request body     api.StopJobAPIRequest true "All fields required"
   796→// @success     200     {object} schema.Job                 "Success message"
   797→// @failure     400     {object} api.ErrorResponse          "Bad Request"
   798→// @failure     401     {object} api.ErrorResponse          "Unauthorized"
   799→// @failure     403     {object} api.ErrorResponse          "Forbidden"
   800→// @failure     404     {object} api.ErrorResponse          "Resource not found"
   801→// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: job has already been stopped"
   802→// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
   803→// @security    ApiKeyAuth
   804→// @router      /api/jobs/stop_job/ [post]
   805→func (api *RestAPI) stopJobByRequest(rw http.ResponseWriter, r *http.Request) {
   806→	// Parse request body
   807→	req := StopJobAPIRequest{}
   808→	if err := decode(r.Body, &req); err != nil {
   809→		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
   810→		return
   811→	}
   812→
   813→	// Fetch job (that will be stopped) from db
   814→	var job *schema.Job
   815→	var err error
   816→	if req.JobID == nil {


   805→func (api *RestAPI) stopJobByRequest(rw http.ResponseWriter, r *http.Request) {
   806→	// Parse request body
   807→	req := StopJobAPIRequest{}
   808→	if err := decode(r.Body, &req); err != nil {
   809→		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
   810→		return
   811→	}
   812→
   813→	// Fetch job (that will be stopped) from db
   814→	var job *schema.Job
   815→	var err error
   816→	if req.JobID == nil {
   817→		handleError(errors.New("the field 'jobId' is required"), http.StatusBadRequest, rw)
   818→		return
   819→	}
   820→
   821→	isCached := false
   822→	job, err = api.JobRepository.FindCached(req.JobID, req.Cluster, req.StartTime)
   823→	if err != nil {
   824→		// Not in cache, try main job table
   825→		job, err = api.JobRepository.Find(req.JobID, req.Cluster, req.StartTime)
   826→		if err != nil {
   827→			handleError(fmt.Errorf("finding job failed: %w", err), http.StatusNotFound, rw)
   828→			return
   829→		}
   830→	} else {
   831→		isCached = true
   832→	}
   833→
   834→	api.checkAndHandleStopJob(rw, job, req, isCached)
   835→}
   836→
   837→// deleteJobByID godoc
   838→// @summary     Remove a job from the sql database
   839→// @tags Job remove
   840→// @description Job to remove is specified by database ID. This will not remove the job from the job archive.
   841→// @produce     json
   842→// @param       id      path     int                   true "Database ID of Job"
   843→// @success     200     {object} api.DefaultAPIResponse  "Success message"
   844→// @failure     400     {object} api.ErrorResponse          "Bad Request"
   845→// @failure     401     {object} api.ErrorResponse          "Unauthorized"
   846→// @failure     403     {object} api.ErrorResponse          "Forbidden"
   847→// @failure     404     {object} api.ErrorResponse          "Resource not found"
   848→// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: finding job failed: sql: no rows in result set"
   849→// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
   850→// @security    ApiKeyAuth
   851→// @router      /api/jobs/delete_job/{id} [delete]
   852→func (api *RestAPI) deleteJobByID(rw http.ResponseWriter, r *http.Request) {
   853→	// Fetch job (that will be stopped) from db
   854→	id := chi.URLParam(r, "id")
   855→	var err error
   856→	if id != "" {
   857→		id, e := strconv.ParseInt(id, 10, 64)
   858→		if e != nil {
   859→			handleError(fmt.Errorf("integer expected in path for id: %w", e), http.StatusBadRequest, rw)
   860→			return
   861→		}
   862→
   863→		err = api.JobRepository.DeleteJobByID(id)
   864→	} else {
   865→		handleError(errors.New("the parameter 'id' is required"), http.StatusBadRequest, rw)
   866→		return
   867→	}
   868→	if err != nil {
   869→		handleError(fmt.Errorf("deleting job failed: %w", err), http.StatusUnprocessableEntity, rw)
   870→		return
   871→	}
   872→	rw.Header().Add("Content-Type", "application/json")
   873→	rw.WriteHeader(http.StatusOK)
   874→	if err := json.NewEncoder(rw).Encode(DefaultAPIResponse{
   875→		Message: fmt.Sprintf("Successfully deleted job %s", id),
   876→	}); err != nil {
   877→		cclog.Errorf("Failed to encode response: %v", err)
   878→	}
   879→}
   880→
   881→// deleteJobByRequest godoc
   882→// @summary     Remove a job from the sql database
   883→// @tags Job remove
   884→// @description Job to delete is specified by request body. All fields are required in this case.
   885→// @accept      json
   886→// @produce     json
   887→// @param       request body     api.DeleteJobAPIRequest true "All fields required"
   888→// @success     200     {object} api.DefaultAPIResponse  "Success message"
   889→// @failure     400     {object} api.ErrorResponse          "Bad Request"
   890→// @failure     401     {object} api.ErrorResponse          "Unauthorized"
   891→// @failure     403     {object} api.ErrorResponse          "Forbidden"
   892→// @failure     404     {object} api.ErrorResponse          "Resource not found"
   893→// @failure     422     {object} api.ErrorResponse          "Unprocessable Entity: finding job failed: sql: no rows in result set"
   894→// @failure     500     {object} api.ErrorResponse          "Internal Server Error"
   895→// @security    ApiKeyAuth
   896→// @router      /api/jobs/delete_job/ [delete]
   897→func (api *RestAPI) deleteJobByRequest(rw http.ResponseWriter, r *http.Request) {
   898→	// Parse request body
   899→	req := DeleteJobAPIRequest{}
   900→	if err := decode(r.Body, &req); err != nil {
   901→		handleError(fmt.Errorf("parsing request body failed: %w", err), http.StatusBadRequest, rw)
   902→		return
   903→	}
   904→


Bash

grep -n "func Retention\|func WALStaging\|func MemoryUsageTracker" /Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/*.go


/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go:320:func Retention(wg *sync.WaitGroup, ctx context.Context) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/metricstore.go:383:func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
/Users/jan/prg/ClusterCockpit/cc-backend/pkg/metricstore/walCheckpoint.go:118:func WALStaging(wg *sync.WaitGroup, ctx context.Context) {


internal/repository/job.go-192-		return fmt.Errorf("failed to optimize database: %w", err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-193-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-194-	return nil
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-195-}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-196-
internal/repository/job.go:197:// Flush removes all data from job-related tables (jobtag, tag, job).
internal/repository/job.go-198-/ WARNING: This is a destructive operation that deletes all job data.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-199-// Use with extreme caution, typically only for testing or complete resets.
internal/repository/job.go:200:func (r *JobRepository) Flush() error {
internal/repository/job.go-201-	if _, err := r.DB.Exec(`DELETE FROM jobtag`); err != nil {
internal/repository/job.go-202-		cclog.Errorf("Error while deleting from jobtag table: %v", err)
internal/repository/job.go-203-		return fmt.Errorf("failed to flush jobtag table: %w", err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-204-	}
internal/repository/job.go-205-	if _, err := r.DB.Exec(`DELETE FROM tag`); err != nil {
--
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-253-	r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-254-	cclog.Debugf("Timer FetchMetadata %s", time.Since(start))
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-255-	return job.MetaData, nil
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-256-}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-257-
internal/repository/job.go:258:// UpdateMetadata adds or updates a single metadata key-value pair for a job.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-259-// The entire metadata map is re-marshaled and stored, and the cache is invalidated.
internal/repository/job.go:260:// Also triggers archive metadata update via archive.UpdateMetadata.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-261-//
internal/repository/job.go-262-/ Parameters:
internal/repository/job.go-263-/   - job: Job struct with valid ID, existing metadata will be fetched if not present
internal/repository/job.go-264-/   - key: Metadata key to set
internal/repository/job.go-265-/   - val: Metadata value to set
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-266-//
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-267-// Returns an error if the job is nil, metadata fetch fails, or database update fails.
internal/repository/job.go:268:func (r *JobRepository) UpdateMetadata(job *schema.Job, key, val string) (err error) {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-269-	if job == nil {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-270-		return fmt.Errorf("job cannot be nil")
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-271-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-272-
internal/repository/job.go-273-	cachekey := fmt.Sprintf("metadata:%d", *job.ID)
--
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-300-		cclog.Warnf("Error while updating metadata for job, DB ID '%v'", *job.ID)
internal/repository/job.go-301-		return fmt.Errorf("failed to update metadata in database for job %d: %w", *job.ID, err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-302-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-303-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-304-	r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour)
internal/repository/job.go:305:	return archive.UpdateMetadata(job, job.MetaData)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-306-}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-307-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-308-// FetchFootprint retrieves and unmarshals the performance footprint JSON for a job.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-309-// Unlike FetchMetadata, footprints are NOT cached as they can be large and change frequently.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-310-//
--
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-381-	r.cache.Put(cachekey, job.EnergyFootprint, len(job.EnergyFootprint), 24*time.Hour)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-382-	cclog.Debugf("Timer FetchEnergyFootprint %s", time.Since(start))
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-383-	return job.EnergyFootprint, nil
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-384-}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-385-
internal/repository/job.go:386:// DeleteJobsBefore removes jobs older than the specified start time.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-387-// Optionally preserves tagged jobs to protect important data from deletion.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-388-// Cache entries for deleted jobs are automatically invalidated.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-389-//
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-390-// This is typically used for data retention policies and cleanup operations.
internal/repository/job.go-391-/ WARNING: This is a destructive operation that permanently deletes job records.
--
internal/repository/job.go-394-/   - startTime: Unix timestamp, jobs with start_time < this value will be deleted
internal/repository/job.go-395-/   - omitTagged: "none" = delete all jobs, "all" = skip any tagged jobs,
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-396-//     "user" = skip jobs with user-created tags (not auto-tagger types "app"/"jobClass")
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-397-//
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-398-// Returns the count of deleted jobs or an error if the operation fails.
internal/repository/job.go:399:func (r *JobRepository) DeleteJobsBefore(startTime int64, omitTagged string) (int, error) {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-400-	var cnt int
internal/repository/job.go-401-	q := sq.Select("count(*)").From("job").Where("job.start_time < ?", startTime)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-402-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-403-	switch omitTagged {
internal/repository/job.go-404-	case "all":
--
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-451-	}
internal/repository/job.go-452-	_, err := qd.RunWith(r.DB).Exec()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-453-
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-454-	if err != nil {
internal/repository/job.go-455-		s, _, _ := qd.ToSql()
internal/repository/job.go:456:		cclog.Errorf(" DeleteJobsBefore(%d) with %s: error %#v", startTime, s, err)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-457-	} else {
internal/repository/job.go:458:		cclog.Debugf("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt)
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-459-	}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-460-	return cnt, err
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-461-}
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-462-
internal/repository/job.go:463:// DeleteJobByID permanently removes a single job by its database ID.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-464-// Cache entries for the deleted job are automatically invalidated.
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-465-//
internal/repository/job.go-466-/ Parameters:
internal/repository/job.go-467-/   - id: Database ID (primary key) of the job to delete
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-468-//
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-469-// Returns an error if the deletion fails.
internal/repository/job.go:470:func (r *JobRepository) DeleteJobByID(id int64) error {
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-471-	// Invalidate cache entries before deletion
internal/repository/job.go-472-	r.cache.Del(fmt.Sprintf("metadata:%d", id))
internal/repository/job.go-473-	r.cache.Del(fmt.Sprintf("energyFootprint:%d", id))
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-474-
internal/repository/job.go-475-	qd := sq.Delete("job").Where("job.id = ?", id)
internal/repository/job.go-476-	_, err := qd.RunWith(r.DB).Exec()
/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go-477-

[Showing results with pagination = limit: 100, offset: 0]


   320→func Retention(wg *sync.WaitGroup, ctx context.Context) {
   321→	ms := GetMemoryStore()
   322→
   323→	wg.Go(func() {
   324→		d, err := time.ParseDuration(Keys.RetentionInMemory)
   325→		if err != nil {
   326→			cclog.Fatal(err)
   327→		}
   328→		if d <= 0 {
   329→			return
   330→		}
   331→
   332→		tickInterval := d / 2
   333→		if tickInterval <= 0 {
   334→			return
   335→		}
   336→		ticker := time.NewTicker(tickInterval)
   337→		defer ticker.Stop()
   338→
   339→		for {
   340→			select {
   341→			case <-ctx.Done():
   342→				return
   343→			case <-ticker.C:
   344→				state.mu.Lock()
   345→
   346→				t := time.Now().Add(-d)
   347→
   348→				state.lastRetentionTime = t.Unix()
   349→
   350→				cclog.Infof("[METRICSTORE]> start freeing buffers (older than %s)...\n", t.Format(time.RFC3339))
   351→
   352→				freed, err := Free(ms, t)
   353→				if err != nil {
   354→					cclog.Errorf("[METRICSTORE]> freeing up buffers failed: %s\n", err.Error())
   355→				} else {
   356→					cclog.Infof("[METRICSTORE]> done: %d buffers freed\n", freed)
   357→				}
   358→
   359→				state.mu.Unlock()
   360→
   361→				// Clean up the buffer pool
   362→				bufferPool.Clean(state.lastRetentionTime)
   363→			}
   364→		}
   365→	})
   366→}
   367→
   368→// MemoryUsageTracker starts a background goroutine that monitors memory usage.
   369→//
   370→// This worker checks actual process memory usage (via runtime.MemStats) periodically
   371→// and force-frees buffers if memory exceeds the configured cap. It uses FreeOSMemory()
   372→// to return memory to the OS after freeing buffers, avoiding aggressive GC that causes
   373→// performance issues.
   374→//
   375→// The tracker logs both actual memory usage (heap allocated) and metric data size for
   376→// visibility into memory overhead from Go runtime structures and allocations.
   377→//
   378→// Parameters:
   379→//   - wg: WaitGroup to signal completion when context is cancelled
   380→//   - ctx: Context for cancellation signal
   381→//
   382→// The goroutine exits when ctx is cancelled.
   383→func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
   384→	ms := GetMemoryStore()
   385→
   386→	wg.Go(func() {
   387→		normalInterval := DefaultMemoryUsageTrackerInterval
   388→		fastInterval := 30 * time.Second
   389→
   390→		if normalInterval <= 0 {
   391→			return
   392→		}
   393→
   394→		ticker := time.NewTicker(normalInterval)
   395→		defer ticker.Stop()
   396→		currentInterval := normalInterval
   397→
   398→		for {
   399→			select {
   400→			case <-ctx.Done():
   401→				return
   402→			case <-ticker.C:
   403→				var mem runtime.MemStats
   404→				runtime.ReadMemStats(&mem)
   405→				actualMemoryGB := float64(mem.Alloc) / 1e9
   406→				metricDataGB := ms.SizeInGB()
   407→				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
   408→
   409→				freedExcluded := 0
   410→				freedEmergency := 0
   411→				var err error
   412→
   413→				state.mu.RLock()
   414→				lastRetention := state.lastRetentionTime
   415→				selectorsExcluded := state.selectorsExcluded
   416→				state.mu.RUnlock()
   417→
   418→				if lastRetention != 0 && selectorsExcluded {
   419→					freedExcluded, err = ms.Free(nil, lastRetention)


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file contains job statistics and histogram generation functionality for the JobRepository.
     7→//
     8→// # Job Statistics
     9→//
    10→// The statistics methods provide aggregated metrics about jobs including total jobs, users,
    11→// walltime, and resource usage (nodes, cores, accelerators). Statistics can be computed:
    12→//   - Overall (JobsStats): Single aggregate across all matching jobs
    13→//   - Grouped (JobsStatsGrouped): Aggregated by user, project, cluster, or subcluster
    14→//   - Counts (JobCountGrouped, AddJobCount): Simple job counts with optional filtering
    15→//
    16→// All statistics methods support filtering via JobFilter and respect security contexts.
    17→//
    18→// # Histograms
    19→//
    20→// Histogram methods generate distribution data for visualization:
    21→//   - Duration, nodes, cores, accelerators (AddHistograms)
    22→//   - Job metrics like CPU load, memory usage (AddMetricHistograms)
    23→//
    24→// Histograms use intelligent binning:
    25→//   - Duration: Variable bin sizes (1m, 10m, 1h, 6h, 12h, 24h) with zero-padding
    26→//   - Resources: Natural value-based bins
    27→//   - Metrics: Normalized to peak values with configurable bin counts
    28→//
    29→// # Running vs. Completed Jobs
    30→//
    31→// Statistics handle running jobs specially:
    32→//   - Duration calculated as (now - start_time) for running jobs
    33→//   - Metric histograms for running jobs load data from metric backend instead of footprint
    34→//   - Job state filtering distinguishes running/completed jobs
    35→//
    36→// # Performance Considerations
    37→//
    38→// - All queries use prepared statements via stmtCache
    39→// - Complex aggregations use SQL for efficiency
    40→// - Histogram pre-initialization ensures consistent bin ranges
    41→// - Metric histogram queries limited to 5000 jobs for running job analysis
    42→
    43→package repository
    44→
    45→import (
    46→	"context"
    47→	"database/sql"
    48→	"fmt"
    49→	"time"
    50→
    51→	"github.com/ClusterCockpit/cc-backend/internal/config"
    52→	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
    53→	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
    54→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    55→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    56→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    57→	sq "github.com/Masterminds/squirrel"
    58→	"golang.org/x/sync/errgroup"
    59→)
    60→
    61→// groupBy2column maps GraphQL Aggregate enum values to their corresponding database column names.
    62→// Used by JobsStatsGrouped and JobCountGrouped to translate user-facing grouping dimensions
    63→// into SQL GROUP BY clauses. GraphQL validation ensures only valid enum values are accepted.
    64→var groupBy2column = map[model.Aggregate]string{
    65→	model.AggregateUser:       "job.hpc_user",
    66→	model.AggregateProject:    "job.project",
    67→	model.AggregateCluster:    "job.cluster",
    68→	model.AggregateSubcluster: "job.subcluster",
    69→}
    70→
    71→// sortBy2column maps GraphQL SortByAggregate enum values to their corresponding computed column names.
    72→// Used by JobsStatsGrouped to translate sort preferences into SQL ORDER BY clauses.
    73→// Column names match the AS aliases used in buildStatsQuery.
    74→var sortBy2column = map[model.SortByAggregate]string{
    75→	model.SortByAggregateTotaljobs:      "totalJobs",
    76→	model.SortByAggregateTotalusers:     "totalUsers",
    77→	model.SortByAggregateTotalwalltime:  "totalWalltime",
    78→	model.SortByAggregateTotalnodes:     "totalNodes",
    79→	model.SortByAggregateTotalnodehours: "totalNodeHours",
    80→	model.SortByAggregateTotalcores:     "totalCores",
    81→	model.SortByAggregateTotalcorehours: "totalCoreHours",
    82→	model.SortByAggregateTotalaccs:      "totalAccs",
    83→	model.SortByAggregateTotalacchours:  "totalAccHours",
    84→}
    85→
    86→// buildCountQuery constructs a SQL query to count jobs with optional grouping and filtering.
    87→//
    88→// Parameters:
    89→//   - filter: Job filters to apply (cluster, user, time range, etc.)
    90→//   - kind: Special filter - "running" for running jobs only, "short" for jobs under threshold
    91→//   - col: Column name to GROUP BY; empty string for total count without grouping
    92→//
    93→// Returns a SelectBuilder that produces either:
    94→//   - Single count: COUNT(job.id) when col is empty
    95→//   - Grouped counts: col, COUNT(job.id) when col is specified
    96→//
    97→// The kind parameter enables counting specific job categories:
    98→//   - "running": Only jobs with job_state = 'running'
    99→//   - "short": Only jobs with duration < ShortRunningJobsDuration config value
   100→//   - empty: All jobs matching filters
   101→func (r *JobRepository) buildCountQuery(
   102→	filter []*model.JobFilter,
   103→	kind string,
   104→	col string,
   105→) sq.SelectBuilder {
   106→	var query sq.SelectBuilder
   107→
   108→	if col != "" {
   109→		query = sq.Select(col, "COUNT(*)").From("job").GroupBy(col)
   110→	} else {
   111→		query = sq.Select("COUNT(*)").From("job")
   112→	}
   113→
   114→	switch kind {
   115→	case "running":
   116→		query = query.Where("job.job_state = 'running'")
   117→	case "short":
   118→		query = query.Where("job.duration < ?", config.Keys.ShortRunningJobsDuration)
   119→	}
   120→
   121→	for _, f := range filter {
   122→		query = BuildWhereClause(f, query)
   123→	}
   124→
   125→	return query
   126→}
   127→
   128→// buildStatsQuery constructs a SQL query to compute job statistics with optional grouping.
   129→// Only requested columns are computed; unrequested columns select 0 as placeholder.
   130→//
   131→// Parameters:
   132→//   - filter: Job filters to apply (cluster, user, time range, etc.)
   133→//   - col: Column name to GROUP BY; empty string for overall statistics without grouping
   134→//   - shortThreshold: Duration threshold in seconds for counting short-running jobs
   135→//   - reqFields: Set of requested field names; nil means compute all fields
   136→func (r *JobRepository) buildStatsQuery(
   137→	filter []*model.JobFilter,
   138→	col string,
   139→	shortThreshold int,
   140→	reqFields map[string]bool,
   141→) sq.SelectBuilder {
   142→	now := time.Now().Unix()
   143→
   144→	// Helper: return real expression if field is requested (or reqFields is nil), else "0 as alias"
   145→	need := func(field string) bool {
   146→		return reqFields == nil || reqFields[field]
   147→	}
   148→	durationExpr := fmt.Sprintf(`(CASE WHEN job.job_state = 'running' THEN %d - job.start_time ELSE job.duration END)`, now)
   149→
   150→	// Build column list
   151→	columns := make([]string, 0, 14)
   152→
   153→	if col != "" {
   154→		columns = append(columns, col)
   155→	}
   156→
   157→	columns = append(columns, "COUNT(*) as totalJobs")
   158→
   159→	if need("totalUsers") && col != "job.hpc_user" {
   160→		columns = append(columns, "COUNT(DISTINCT job.hpc_user) AS totalUsers")
   161→	} else {
   162→		columns = append(columns, "0 AS totalUsers")
   163→	}
   164→
   165→	if need("totalWalltime") {
   166→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s) / 3600) as int) as totalWalltime`, durationExpr))
   167→	} else {
   168→		columns = append(columns, "0 as totalWalltime")
   169→	}
   170→
   171→	if need("totalNodes") {
   172→		columns = append(columns, `CAST(SUM(job.num_nodes) as int) as totalNodes`)
   173→	} else {
   174→		columns = append(columns, "0 as totalNodes")
   175→	}
   176→
   177→	if need("totalNodeHours") {
   178→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s * job.num_nodes) / 3600) as int) as totalNodeHours`, durationExpr))
   179→	} else {
   180→		columns = append(columns, "0 as totalNodeHours")
   181→	}
   182→
   183→	if need("totalCores") {
   184→		columns = append(columns, `CAST(SUM(job.num_hwthreads) as int) as totalCores`)
   185→	} else {
   186→		columns = append(columns, "0 as totalCores")
   187→	}
   188→
   189→	if need("totalCoreHours") {
   190→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s * job.num_hwthreads) / 3600) as int) as totalCoreHours`, durationExpr))
   191→	} else {
   192→		columns = append(columns, "0 as totalCoreHours")
   193→	}
   194→
   195→	if need("totalAccs") {
   196→		columns = append(columns, `CAST(SUM(job.num_acc) as int) as totalAccs`)
   197→	} else {
   198→		columns = append(columns, "0 as totalAccs")
   199→	}
   200→
   201→	if need("totalAccHours") {
   202→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s * job.num_acc) / 3600) as int) as totalAccHours`, durationExpr))
   203→	} else {
   204→		columns = append(columns, "0 as totalAccHours")
   205→	}
   206→
   207→	if need("runningJobs") {
   208→		columns = append(columns, `COUNT(CASE WHEN job.job_state = 'running' THEN 1 END) as runningJobs`)
   209→	} else {
   210→		columns = append(columns, "0 as runningJobs")
   211→	}
   212→
   213→	if need("shortJobs") {
   214→		columns = append(columns, fmt.Sprintf(`COUNT(CASE WHEN job.duration < %d THEN 1 END) as shortJobs`, shortThreshold))
   215→	} else {
   216→		columns = append(columns, "0 as shortJobs")
   217→	}
   218→
   219→	query := sq.Select(columns...).From("job")
   220→	if col != "" {
   221→		query = query.GroupBy(col)
   222→	}
   223→
   224→	for _, f := range filter {
   225→		query = BuildWhereClause(f, query)
   226→	}
   227→
   228→	return query
   229→}
   230→
   231→// JobsStatsGrouped computes job statistics grouped by a dimension (user, project, cluster, or subcluster).
   232→// Only columns listed in reqFields are computed; others return 0. User display names are looked up
   233→// in a separate lightweight query to avoid JOIN overhead on the main aggregation.
   234→func (r *JobRepository) JobsStatsGrouped(
   235→	ctx context.Context,
   236→	filter []*model.JobFilter,
   237→	page *model.PageRequest,
   238→	sortBy *model.SortByAggregate,
   239→	groupBy *model.Aggregate,
   240→	reqFields map[string]bool,
   241→) ([]*model.JobsStatistics, error) {
   242→	col := groupBy2column[*groupBy]
   243→	query := r.buildStatsQuery(filter, col, config.Keys.ShortRunningJobsDuration, reqFields)
   244→
   245→	query, err := SecurityCheck(ctx, query)
   246→	if err != nil {
   247→		return nil, err
   248→	}
   249→
   250→	if sortBy != nil {
   251→		sortBy := sortBy2column[*sortBy]
   252→		query = query.OrderBy(fmt.Sprintf("%s DESC", sortBy))
   253→	}
   254→	if page != nil && page.ItemsPerPage != -1 {
   255→		limit := uint64(page.ItemsPerPage)
   256→		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
   257→	}
   258→
   259→	rows, err := query.RunWith(r.DB).QueryContext(ctx)
   260→	if err != nil {
   261→		cclog.Warn("Error while querying DB for job statistics")
   262→		return nil, err
   263→	}
   264→	defer rows.Close()
   265→
   266→	stats := make([]*model.JobsStatistics, 0, 100)
   267→
   268→	for rows.Next() {
   269→		var id sql.NullString
   270→		var jobs, users, walltime, nodes, nodeHours, cores, coreHours, accs, accHours, runningJobs, shortJobs sql.NullInt64
   271→		if err := rows.Scan(&id, &jobs, &users, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours, &runningJobs, &shortJobs); err != nil {
   272→			cclog.Warnf("Error while scanning rows: %s", err.Error())
   273→			return nil, err
   274→		}
   275→
   276→		if id.Valid {
   277→			stats = append(stats,
   278→				&model.JobsStatistics{
   279→					ID:             id.String,
   280→					TotalJobs:      int(jobs.Int64),
   281→					TotalUsers:     int(users.Int64),
   282→					TotalWalltime:  int(walltime.Int64),
   283→					TotalNodes:     int(nodes.Int64),
   284→					TotalNodeHours: int(nodeHours.Int64),
   285→					TotalCores:     int(cores.Int64),
   286→					TotalCoreHours: int(coreHours.Int64),
   287→					TotalAccs:      int(accs.Int64),
   288→					TotalAccHours:  int(accHours.Int64),
   289→					RunningJobs:    int(runningJobs.Int64),
   290→					ShortJobs:      int(shortJobs.Int64),
   291→				})
   292→		}
   293→	}
   294→
   295→	if err := rows.Err(); err != nil {
   296→		return nil, err
   297→	}
   298→
   299→	// Post-query name lookup for user grouping (avoids LEFT JOIN on aggregation query)
   300→	if col == "job.hpc_user" && len(stats) > 0 {
   301→		usernames := make([]any, len(stats))
   302→		for i, s := range stats {
   303→			usernames[i] = s.ID
   304→		}
   305→
   306→		nameQuery := sq.Select("username", "name").From("hpc_user").Where(sq.Eq{"username": usernames})
   307→		nameRows, err := nameQuery.RunWith(r.DB).QueryContext(ctx)
   308→		if err != nil {
   309→			cclog.Warnf("Error looking up user names: %s", err.Error())
   310→			// Non-fatal: stats are still valid without display names
   311→		} else {
   312→			defer nameRows.Close()
   313→			nameMap := make(map[string]string, len(stats))
   314→			for nameRows.Next() {
   315→				var username, name string
   316→				if err := nameRows.Scan(&username, &name); err == nil {
   317→					nameMap[username] = name
   318→				}
   319→			}
   320→			for _, s := range stats {
   321→				if name, ok := nameMap[s.ID]; ok {
   322→					s.Name = name
   323→				}
   324→			}
   325→		}
   326→	}
   327→
   328→	return stats, nil
   329→}
   330→
   331→// JobsStats computes overall job statistics across all matching jobs without grouping.
   332→//
   333→// This method provides a single aggregate view of job metrics, useful for dashboard
   334→// summaries and overall system utilization reports.
   335→//
   336→// Parameters:
   337→//   - ctx: Context for security checks and cancellation
   338→//   - filter: Filters to apply (time range, cluster, job state, etc.)
   339→//
   340→// Returns a single-element slice containing aggregate statistics:
   341→//   - totalJobs, totalUsers, totalWalltime
   342→//   - totalNodeHours, totalCoreHours, totalAccHours
   343→//
   344→// Unlike JobsStatsGrouped, this returns overall totals without breaking down by dimension.
   345→// Security checks are applied via SecurityCheck to respect user access levels.
   346→func (r *JobRepository) JobsStats(
   347→	ctx context.Context,
   348→	filter []*model.JobFilter,
   349→	reqFields map[string]bool,
   350→) ([]*model.JobsStatistics, error) {
   351→	start := time.Now()
   352→	query := r.buildStatsQuery(filter, "", config.Keys.ShortRunningJobsDuration, reqFields)
   353→	query, err := SecurityCheck(ctx, query)
   354→	if err != nil {
   355→		return nil, err
   356→	}
   357→
   358→	row := query.RunWith(r.DB).QueryRowContext(ctx)
   359→	stats := make([]*model.JobsStatistics, 0, 1)
   360→
   361→	var jobs, users, walltime, nodes, nodeHours, cores, coreHours, accs, accHours, runningJobs, shortJobs sql.NullInt64
   362→	if err := row.Scan(&jobs, &users, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours, &runningJobs, &shortJobs); err != nil {
   363→		cclog.Warn("Error while scanning rows")
   364→		return nil, err
   365→	}
   366→
   367→	if jobs.Valid {
   368→		var totalNodeHours, totalCoreHours, totalAccHours int
   369→
   370→		if nodeHours.Valid {
   371→			totalNodeHours = int(nodeHours.Int64)
   372→		}
   373→		if coreHours.Valid {
   374→			totalCoreHours = int(coreHours.Int64)
   375→		}
   376→		if accHours.Valid {
   377→			totalAccHours = int(accHours.Int64)
   378→		}
   379→		stats = append(stats,
   380→			&model.JobsStatistics{
   381→				TotalJobs:      int(jobs.Int64),
   382→				TotalUsers:     int(users.Int64),
   383→				TotalWalltime:  int(walltime.Int64),
   384→				TotalNodeHours: totalNodeHours,
   385→				TotalCoreHours: totalCoreHours,
   386→				TotalAccHours:  totalAccHours,
   387→				RunningJobs:    int(runningJobs.Int64),
   388→				ShortJobs:      int(shortJobs.Int64),
   389→			})
   390→	}
   391→
   392→	cclog.Debugf("Timer JobStats %s", time.Since(start))
   393→	return stats, nil
   394→}
   395→
   396→// LoadJobStat retrieves a specific statistic for a metric from a job's statistics.
   397→// Returns 0.0 if the metric is not found or statType is invalid.
   398→//
   399→// Parameters:
   400→//   - job: Job struct with populated Statistics field
   401→//   - metric: Name of the metric to query (e.g., "cpu_load", "mem_used")
   402→//   - statType: Type of statistic: "avg", "min", or "max"
   403→//
   404→// Returns the requested statistic value or 0.0 if not found.
   405→func LoadJobStat(job *schema.Job, metric string, statType string) float64 {
   406→	if stats, ok := job.Statistics[metric]; ok {
   407→		switch statType {
   408→		case "avg":
   409→			return stats.Avg
   410→		case "max":
   411→			return stats.Max
   412→		case "min":
   413→			return stats.Min
   414→		default:
   415→			cclog.Errorf("Unknown stat type %s", statType)
   416→		}
   417→	}
   418→
   419→	return 0.0
   420→}
   421→
   422→// JobCountGrouped counts jobs grouped by a dimension without computing detailed statistics.
   423→//
   424→// This is a lightweight alternative to JobsStatsGrouped when only job counts are needed,
   425→// avoiding the overhead of calculating walltime and resource usage metrics.
   426→//
   427→// Parameters:
   428→//   - ctx: Context for security checks
   429→//   - filter: Filters to apply
   430→//   - groupBy: Grouping dimension (User, Project, Cluster, or SubCluster)
   431→//
   432→// Returns JobsStatistics with only ID and TotalJobs populated for each group.
   433→func (r *JobRepository) JobCountGrouped(
   434→	ctx context.Context,
   435→	filter []*model.JobFilter,
   436→	groupBy *model.Aggregate,
   437→) ([]*model.JobsStatistics, error) {
   438→	start := time.Now()
   439→	col := groupBy2column[*groupBy]
   440→	query := r.buildCountQuery(filter, "", col)
   441→	query, err := SecurityCheck(ctx, query)
   442→	if err != nil {
   443→		return nil, err
   444→	}
   445→	rows, err := query.RunWith(r.DB).QueryContext(ctx)
   446→	if err != nil {
   447→		cclog.Warn("Error while querying DB for job statistics")
   448→		return nil, err
   449→	}
   450→	defer rows.Close()
   451→
   452→	stats := make([]*model.JobsStatistics, 0, 100)
   453→
   454→	for rows.Next() {
   455→		var id sql.NullString
   456→		var cnt sql.NullInt64
   457→		if err := rows.Scan(&id, &cnt); err != nil {
   458→			cclog.Warn("Error while scanning rows")
   459→			return nil, err
   460→		}
   461→		if id.Valid {
   462→			stats = append(stats,
   463→				&model.JobsStatistics{
   464→					ID:        id.String,
   465→					TotalJobs: int(cnt.Int64),
   466→				})
   467→		}
   468→	}
   469→
   470→	if err := rows.Err(); err != nil {
   471→		return nil, err
   472→	}
   473→
   474→	cclog.Debugf("Timer JobCountGrouped %s", time.Since(start))
   475→	return stats, nil
   476→}
   477→
   478→// AddJobCountGrouped augments existing statistics with additional job counts by category.
   479→//
   480→// This method enriches JobsStatistics returned by JobsStatsGrouped or JobCountGrouped
   481→// with counts of running or short-running (based on ShortRunningJobsDuration) jobs, matched by group ID.
   482→//
   483→// Parameters:
   484→//   - ctx: Context for security checks
   485→//   - filter: Filters to apply
   486→//   - groupBy: Grouping dimension (must match the dimension used for stats parameter)
   487→//   - stats: Existing statistics to augment (modified in-place by ID matching)
   488→//   - kind: "running" to add RunningJobs count, "short" to add ShortJobs count
   489→//
   490→// Returns the same stats slice with RunningJobs or ShortJobs fields populated per group.
   491→// Groups without matching jobs will have 0 for the added field.
   492→func (r *JobRepository) AddJobCountGrouped(
   493→	ctx context.Context,
   494→	filter []*model.JobFilter,
   495→	groupBy *model.Aggregate,
   496→	stats []*model.JobsStatistics,
   497→	kind string,
   498→) ([]*model.JobsStatistics, error) {
   499→	start := time.Now()
   500→	col := groupBy2column[*groupBy]
   501→	query := r.buildCountQuery(filter, kind, col)
   502→	query, err := SecurityCheck(ctx, query)
   503→	if err != nil {
   504→		return nil, err
   505→	}
   506→	rows, err := query.RunWith(r.DB).QueryContext(ctx)
   507→	if err != nil {
   508→		cclog.Warn("Error while querying DB for job statistics")
   509→		return nil, err
   510→	}
   511→	defer rows.Close()
   512→
   513→	counts := make(map[string]int)
   514→
   515→	for rows.Next() {
   516→		var id sql.NullString
   517→		var cnt sql.NullInt64
   518→		if err := rows.Scan(&id, &cnt); err != nil {
   519→			cclog.Warn("Error while scanning rows")
   520→			return nil, err
   521→		}
   522→		if id.Valid {
   523→			counts[id.String] = int(cnt.Int64)
   524→		}
   525→	}
   526→
   527→	if err := rows.Err(); err != nil {
   528→		return nil, err
   529→	}
   530→
   531→	switch kind {
   532→	case "running":
   533→		for _, s := range stats {
   534→			s.RunningJobs = counts[s.ID]
   535→		}
   536→	case "short":
   537→		for _, s := range stats {
   538→			s.ShortJobs = counts[s.ID]
   539→		}
   540→	}
   541→
   542→	cclog.Debugf("Timer AddJobCountGrouped %s", time.Since(start))
   543→	return stats, nil
   544→}
   545→
   546→// AddJobCount augments existing overall statistics with additional job counts by category.
   547→//
   548→// Similar to AddJobCountGrouped but for ungrouped statistics. Applies the same count
   549→// to all statistics entries (typically just one).
   550→//
   551→// Parameters:
   552→//   - ctx: Context for security checks
   553→//   - filter: Filters to apply
   554→//   - stats: Existing statistics to augment (modified in-place)
   555→//   - kind: "running" to add RunningJobs count, "short" to add ShortJobs count
   556→//
   557→// Returns the same stats slice with RunningJobs or ShortJobs fields set to the total count.
   558→func (r *JobRepository) AddJobCount(
   559→	ctx context.Context,
   560→	filter []*model.JobFilter,
   561→	stats []*model.JobsStatistics,
   562→	kind string,
   563→) ([]*model.JobsStatistics, error) {
   564→	start := time.Now()
   565→	query := r.buildCountQuery(filter, kind, "")
   566→	query, err := SecurityCheck(ctx, query)
   567→	if err != nil {
   568→		return nil, err
   569→	}
   570→	var cnt sql.NullInt64
   571→	if err := query.RunWith(r.DB).QueryRowContext(ctx).Scan(&cnt); err != nil {
   572→		cclog.Warn("Error while querying DB for job count")
   573→		return nil, err
   574→	}
   575→
   576→	count := int(cnt.Int64)
   577→
   578→	switch kind {
   579→	case "running":
   580→		for _, s := range stats {
   581→			s.RunningJobs = count
   582→		}
   583→	case "short":
   584→		for _, s := range stats {
   585→			s.ShortJobs = count
   586→		}
   587→	}
   588→
   589→	cclog.Debugf("Timer AddJobCount %s", time.Since(start))
   590→	return stats, nil
   591→}
   592→
   593→// AddHistograms augments statistics with distribution histograms for job properties.
   594→//
   595→// Generates histogram data for visualization of job duration, node count, core count,
   596→// and accelerator count distributions. Duration histogram uses intelligent binning based
   597→// on the requested resolution.
   598→//
   599→// Parameters:
   600→//   - ctx: Context for security checks
   601→//   - filter: Filters to apply to jobs included in histograms
   602→//   - stat: Statistics struct to augment (modified in-place)
   603→//   - durationBins: Bin size - "1m", "10m", "1h", "6h", "12h", or "24h" (default)
   604→//
   605→// Populates these fields in stat:
   606→//   - HistDuration: Job duration distribution (zero-padded bins)
   607→//   - HistNumNodes: Node count distribution
   608→//   - HistNumCores: Core (hwthread) count distribution
   609→//   - HistNumAccs: Accelerator count distribution
   610→//
   611→// Duration bins are pre-initialized with zeros to ensure consistent ranges for visualization.
   612→// Bin size determines both the width and maximum duration displayed (e.g., "1h" = 48 bins × 1h = 48h max).
   613→func (r *JobRepository) AddHistograms(
   614→	ctx context.Context,
   615→	filter []*model.JobFilter,
   616→	stat *model.JobsStatistics,
   617→	durationBins *string,
   618→) (*model.JobsStatistics, error) {
   619→	start := time.Now()
   620→
   621→	var targetBinCount int
   622→	var targetBinSize int
   623→	switch *durationBins {
   624→	case "1m": // 1 Minute Bins + Max 60 Bins -> Max 60 Minutes
   625→		targetBinCount = 60
   626→		targetBinSize = 60
   627→	case "10m": // 10 Minute Bins + Max 72 Bins -> Max 12 Hours
   628→		targetBinCount = 72
   629→		targetBinSize = 600
   630→	case "1h": // 1 Hour Bins + Max 48 Bins -> Max 48 Hours
   631→		targetBinCount = 48
   632→		targetBinSize = 3600
   633→	case "6h": // 6 Hour Bins + Max 12 Bins -> Max 3 Days
   634→		targetBinCount = 12
   635→		targetBinSize = 21600
   636→	case "12h": // 12 hour Bins + Max 14 Bins -> Max 7 Days
   637→		targetBinCount = 14
   638→		targetBinSize = 43200
   639→	default: // 24h
   640→		targetBinCount = 24
   641→		targetBinSize = 3600
   642→	}
   643→
   644→	// Run all 4 histogram queries in parallel — each writes a distinct struct field.
   645→	g, gctx := errgroup.WithContext(ctx)
   646→
   647→	value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = 'running' THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as int) as value`, time.Now().Unix(), targetBinSize)
   648→
   649→	g.Go(func() error {
   650→		var err error
   651→		stat.HistDuration, err = r.jobsDurationStatisticsHistogram(gctx, value, filter, targetBinSize, &targetBinCount)
   652→		if err != nil {
   653→			cclog.Warn("Error while loading job statistics histogram: job duration")
   654→		}
   655→		return err
   656→	})
   657→	g.Go(func() error {
   658→		var err error
   659→		stat.HistNumNodes, err = r.jobsStatisticsHistogram(gctx, "job.num_nodes as value", filter)
   660→		if err != nil {
   661→			cclog.Warn("Error while loading job statistics histogram: num nodes")
   662→		}
   663→		return err
   664→	})
   665→	g.Go(func() error {
   666→		var err error
   667→		stat.HistNumCores, err = r.jobsStatisticsHistogram(gctx, "job.num_hwthreads as value", filter)
   668→		if err != nil {
   669→			cclog.Warn("Error while loading job statistics histogram: num hwthreads")
   670→		}
   671→		return err
   672→	})
   673→	g.Go(func() error {
   674→		var err error
   675→		stat.HistNumAccs, err = r.jobsStatisticsHistogram(gctx, "job.num_acc as value", filter)
   676→		if err != nil {
   677→			cclog.Warn("Error while loading job statistics histogram: num acc")
   678→		}
   679→		return err
   680→	})
   681→
   682→	if err := g.Wait(); err != nil {
   683→		return nil, err
   684→	}
   685→
   686→	cclog.Debugf("Timer AddHistograms %s", time.Since(start))
   687→	return stat, nil
   688→}
   689→
   690→// AddMetricHistograms augments statistics with distribution histograms for job metrics.
   691→//
   692→// Generates histogram data for metrics like CPU load, memory usage, etc. Handles running
   693→// and completed jobs differently: running jobs load data from metric backend, completed jobs
   694→// use footprint data from database.
   695→//
   696→// Parameters:
   697→//   - ctx: Context for security checks
   698→//   - filter: Filters to apply (MUST contain State filter for running jobs)
   699→//   - metrics: List of metric names to histogram (e.g., ["cpu_load", "mem_used"])
   700→//   - stat: Statistics struct to augment (modified in-place)
   701→//   - targetBinCount: Number of histogram bins (default: 10)
   702→//
   703→// Populates HistMetrics field in stat with MetricHistoPoints for each metric.
   704→//
   705→// Binning algorithm:
   706→//   - Values normalized to metric's peak value from cluster configuration
   707→//   - Bins evenly distributed from 0 to peak
   708→//   - Pre-initialized with zeros for consistent visualization
   709→//
   710→// Limitations:
   711→//   - Running jobs: Limited to 5000 jobs for performance
   712→//   - Requires valid cluster configuration with metric peak values
   713→//   - Uses footprint statistic (avg/max/min) configured per metric
   714→func (r *JobRepository) AddMetricHistograms(
   715→	ctx context.Context,
   716→	filter []*model.JobFilter,
   717→	metrics []string,
   718→	stat *model.JobsStatistics,
   719→	targetBinCount *int,
   720→) (*model.JobsStatistics, error) {
   721→	start := time.Now()
   722→
   723→	// Running Jobs Only: First query jobdata from sqlite, then query data and make bins
   724→	for _, f := range filter {
   725→		if f.State != nil {
   726→			if len(f.State) == 1 && f.State[0] == "running" {
   727→				stat.HistMetrics = r.runningJobsMetricStatisticsHistogram(ctx, metrics, filter, targetBinCount)
   728→				cclog.Debugf("Timer AddMetricHistograms %s", time.Since(start))
   729→				return stat, nil
   730→			}
   731→		}
   732→	}
   733→
   734→	// All other cases: Query and make bins in sqlite directly
   735→	for _, m := range metrics {
   736→		metricHisto, err := r.jobsMetricStatisticsHistogram(ctx, m, filter, targetBinCount)
   737→		if err != nil {
   738→			cclog.Warnf("Error while loading job metric statistics histogram: %s", m)
   739→			continue
   740→		}
   741→		stat.HistMetrics = append(stat.HistMetrics, metricHisto)
   742→	}
   743→
   744→	cclog.Debugf("Timer AddMetricHistograms %s", time.Since(start))
   745→	return stat, nil
   746→}
   747→
   748→// jobsStatisticsHistogram generates a simple histogram by grouping on a column value.
   749→//
   750→// Used for histograms where the column value directly represents the bin (e.g., node count, core count).
   751→// Unlike duration/metric histograms, this doesn't pre-initialize bins with zeros.
   752→//
   753→// Parameters:
   754→//   - value: SQL expression that produces the histogram value, aliased as "value"
   755→//   - filters: Job filters to apply
   756→//
   757→// Returns histogram points with Value (from column) and Count (number of jobs).
   758→func (r *JobRepository) jobsStatisticsHistogram(
   759→	ctx context.Context,
   760→	value string,
   761→	filters []*model.JobFilter,
   762→) ([]*model.HistoPoint, error) {
   763→	start := time.Now()
   764→	query, qerr := SecurityCheck(ctx,
   765→		sq.Select(value, "COUNT(*) AS count").From("job"))
   766→
   767→	if qerr != nil {
   768→		return nil, qerr
   769→	}
   770→
   771→	for _, f := range filters {
   772→		query = BuildWhereClause(f, query)
   773→	}
   774→
   775→	rows, err := query.GroupBy("value").RunWith(r.DB).QueryContext(ctx)
   776→	if err != nil {
   777→		cclog.Error("Error while running query")
   778→		return nil, err
   779→	}
   780→	defer rows.Close()
   781→
   782→	points := make([]*model.HistoPoint, 0)
   783→	// is it possible to introduce zero values here? requires info about bincount
   784→	for rows.Next() {
   785→		point := model.HistoPoint{}
   786→		if err := rows.Scan(&point.Value, &point.Count); err != nil {
   787→			cclog.Warn("Error while scanning rows")
   788→			return nil, err
   789→		}
   790→
   791→		points = append(points, &point)
   792→	}
   793→
   794→	if err := rows.Err(); err != nil {
   795→		return nil, err
   796→	}
   797→
   798→	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
   799→	return points, nil
   800→}
   801→
   802→// jobsDurationStatisticsHistogram generates a duration histogram with pre-initialized bins.
   803→//
   804→// Bins are zero-padded to provide consistent ranges for visualization, unlike simple
   805→// histograms which only return bins with data. The value parameter should compute
   806→// the bin number from job duration.
   807→//
   808→// Parameters:
   809→//   - value: SQL expression computing bin number from duration, aliased as "value"
   810→//   - filters: Job filters to apply
   811→//   - binSizeSeconds: Width of each bin in seconds
   812→//   - targetBinCount: Number of bins to pre-initialize
   813→//
   814→// Returns histogram points with Value (bin_number × binSizeSeconds) and Count.
   815→// All bins from 1 to targetBinCount are returned, with Count=0 for empty bins.
   816→//
   817→// Algorithm:
   818→//  1. Pre-initialize targetBinCount bins with zero counts
   819→//  2. Query database for actual counts per bin
   820→//  3. Match query results to pre-initialized bins by value
   821→//  4. Bins without matches remain at zero
   822→func (r *JobRepository) jobsDurationStatisticsHistogram(
   823→	ctx context.Context,
   824→	value string,
   825→	filters []*model.JobFilter,
   826→	binSizeSeconds int,
   827→	targetBinCount *int,
   828→) ([]*model.HistoPoint, error) {
   829→	start := time.Now()
   830→	query, qerr := SecurityCheck(ctx,
   831→		sq.Select(value, "COUNT(*) AS count").From("job"))
   832→
   833→	if qerr != nil {
   834→		return nil, qerr
   835→	}
   836→
   837→	// Each bin represents a duration range: bin N = [N*binSizeSeconds, (N+1)*binSizeSeconds)
   838→	// Example: binSizeSeconds=3600 (1 hour), bin 1 = 0-1h, bin 2 = 1-2h, etc.
   839→	points := make([]*model.HistoPoint, 0, *targetBinCount)
   840→	for i := 1; i <= *targetBinCount; i++ {
   841→		point := model.HistoPoint{Value: i * binSizeSeconds, Count: 0}
   842→		points = append(points, &point)
   843→	}
   844→
   845→	// Build a map from bin value (seconds) to slice index for O(1) lookup.
   846→	binMap := make(map[int]int, len(points))
   847→	for i, p := range points {
   848→		binMap[p.Value] = i
   849→	}
   850→
   851→	for _, f := range filters {
   852→		query = BuildWhereClause(f, query)
   853→	}
   854→
   855→	rows, err := query.GroupBy("value").RunWith(r.DB).QueryContext(ctx)
   856→	if err != nil {
   857→		cclog.Error("Error while running query")
   858→		return nil, err
   859→	}
   860→	defer rows.Close()
   861→
   862→	// Match query results to pre-initialized bins using map lookup.
   863→	// point.Value from query is the bin number; multiply by binSizeSeconds to get bin key.
   864→	for rows.Next() {
   865→		point := model.HistoPoint{}
   866→		if err := rows.Scan(&point.Value, &point.Count); err != nil {
   867→			cclog.Warn("Error while scanning rows")
   868→			return nil, err
   869→		}
   870→
   871→		if idx, ok := binMap[point.Value*binSizeSeconds]; ok {
   872→			points[idx].Count = point.Count
   873→		}
   874→	}
   875→
   876→	if err := rows.Err(); err != nil {
   877→		return nil, err
   878→	}
   879→
   880→	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
   881→	return points, nil
   882→}
   883→
   884→// jobsMetricStatisticsHistogram generates a metric histogram using footprint data from completed jobs.
   885→//
   886→// Values are normalized to the metric's peak value and distributed into bins. The algorithm
   887→// is based on SQL histogram generation techniques, extracting metric values from JSON footprint
   888→// and computing bin assignments in SQL.
   889→//
   890→// Parameters:
   891→//   - metric: Metric name (e.g., "cpu_load", "mem_used")
   892→//   - filters: Job filters to apply
   893→//   - bins: Number of bins to generate
   894→//
   895→// Returns MetricHistoPoints with metric name, unit, footprint stat type, and binned data.
   896→//
   897→// Algorithm:
   898→//  1. Determine peak value from cluster configuration (filtered cluster or max across all)
   899→//  2. Generate SQL that extracts footprint value, normalizes to [0,1], multiplies by bin count
   900→//  3. Pre-initialize bins with min/max ranges based on peak value
   901→//  4. Query database for counts per bin
   902→//  5. Match results to pre-initialized bins
   903→//
   904→// Special handling: Values exactly equal to peak are forced into the last bin by multiplying
   905→// peak by 0.999999999 to avoid creating an extra bin.
   906→func (r *JobRepository) jobsMetricStatisticsHistogram(
   907→	ctx context.Context,
   908→	metric string,
   909→	filters []*model.JobFilter,
   910→	bins *int,
   911→) (*model.MetricHistoPoints, error) {
   912→	// Peak value defines the upper bound for binning: values are distributed across
   913→	// bins from 0 to peak. First try to get peak from filtered cluster, otherwise
   914→	// scan all clusters to find the maximum peak value.
   915→	var metricConfig *schema.MetricConfig
   916→	var peak float64
   917→	var unit string
   918→	var footprintStat string
   919→
   920→	// Try to get metric config from filtered cluster
   921→	for _, f := range filters {
   922→		if f.Cluster != nil {
   923→			metricConfig = archive.GetMetricConfig(*f.Cluster.Eq, metric)
   924→			peak = metricConfig.Peak
   925→			unit = metricConfig.Unit.Prefix + metricConfig.Unit.Base
   926→			footprintStat = metricConfig.Footprint
   927→			cclog.Debugf("Cluster %s filter found with peak %f for %s", *f.Cluster.Eq, peak, metric)
   928→		}
   929→	}
   930→
   931→	// If no cluster filter or peak not found, find largest peak across all clusters
   932→	// This ensures histogram can accommodate all possible values
   933→	if peak == 0.0 {
   934→		for _, c := range archive.Clusters {
   935→			for _, m := range c.MetricConfig {
   936→				if m.Name == metric {
   937→					if m.Peak > peak {
   938→						peak = m.Peak
   939→					}
   940→					if unit == "" {
   941→						unit = m.Unit.Prefix + m.Unit.Base
   942→					}
   943→					if footprintStat == "" {
   944→						footprintStat = m.Footprint
   945→					}
   946→				}
   947→			}
   948→		}
   949→	}
   950→
   951→	// Construct SQL histogram bins using normalized values.
   952→	// Algorithm based on: https://jereze.com/code/sql-histogram/ (modified)
   953→	start := time.Now()
   954→
   955→	// Bin calculation formula:
   956→	//   bin_number = CAST( (value / peak) * num_bins AS INTEGER ) + 1
   957→	// Special case: value == peak would create bin N+1, so we test for equality
   958→	// and multiply peak by 0.999999999 to force it into bin N.
   959→	binQuery := fmt.Sprintf(`CAST(
   960→		((case when json_extract(footprint, "$.%s") = %f then %f*0.999999999 else json_extract(footprint, "$.%s") end) / %f)
   961→		* %v as INTEGER )`,
   962→		(metric + "_" + footprintStat), peak, peak, (metric + "_" + footprintStat), peak, *bins)
   963→
   964→	mainQuery := sq.Select(
   965→		fmt.Sprintf(`%s + 1 as bin`, binQuery),
   966→		`count(*) as count`,
   967→	).From("job").Where(
   968→		"JSON_VALID(footprint)",
   969→	).Where(fmt.Sprintf(`json_extract(footprint, "$.%s") is not null and json_extract(footprint, "$.%s") <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak))
   970→
   971→	mainQuery, qerr := SecurityCheck(ctx, mainQuery)
   972→	if qerr != nil {
   973→		return nil, qerr
   974→	}
   975→
   976→	for _, f := range filters {
   977→		mainQuery = BuildWhereClause(f, mainQuery)
   978→	}
   979→
   980→	mainQuery = mainQuery.GroupBy("bin").OrderBy("bin")
   981→
   982→	rows, err := mainQuery.RunWith(r.DB).QueryContext(ctx)
   983→	if err != nil {
   984→		cclog.Errorf("Error while running mainQuery: %s", err)
   985→		return nil, err
   986→	}
   987→	defer rows.Close()
   988→
   989→	// Pre-initialize bins with calculated min/max ranges.
   990→	// Example: peak=1000, bins=10 -> bin 1=[0,100), bin 2=[100,200), ..., bin 10=[900,1000]
   991→	points := make([]*model.MetricHistoPoint, 0, *bins)
   992→	binStep := int(peak) / *bins
   993→	for i := 1; i <= *bins; i++ {
   994→		binMin := (binStep * (i - 1))
   995→		binMax := (binStep * i)
   996→		idx := i
   997→		epoint := model.MetricHistoPoint{Bin: &idx, Count: 0, Min: &binMin, Max: &binMax}
   998→		points = append(points, &epoint)
   999→	}
  1000→
  1001→	// Build a map from bin number to slice index for O(1) lookup.
  1002→	binMap := make(map[int]int, len(points))
  1003→	for i, p := range points {
  1004→		if p.Bin != nil {
  1005→			binMap[*p.Bin] = i
  1006→		}
  1007→	}
  1008→
  1009→	// Match query results to pre-initialized bins using map lookup.
  1010→	for rows.Next() {
  1011→		rpoint := model.MetricHistoPoint{}
  1012→		if err := rows.Scan(&rpoint.Bin, &rpoint.Count); err != nil {
  1013→			cclog.Warnf("Error while scanning rows for %s", metric)
  1014→			return nil, err
  1015→		}
  1016→
  1017→		if rpoint.Bin != nil {
  1018→			if idx, ok := binMap[*rpoint.Bin]; ok {
  1019→				points[idx].Count = rpoint.Count
  1020→			}
  1021→		}
  1022→	}
  1023→
  1024→	if err := rows.Err(); err != nil {
  1025→		return nil, err
  1026→	}
  1027→
  1028→	result := model.MetricHistoPoints{Metric: metric, Unit: unit, Stat: &footprintStat, Data: points}
  1029→
  1030→	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
  1031→	return &result, nil
  1032→}
  1033→
  1034→// runningJobsMetricStatisticsHistogram generates metric histograms for running jobs using live data.
  1035→//
  1036→// Unlike completed jobs which use footprint data from the database, running jobs require
  1037→// fetching current metric averages from the metric backend (via metricdispatch).
  1038→//
  1039→// Parameters:
  1040→//   - metrics: List of metric names
  1041→//   - filters: Job filters (should filter to running jobs only)
  1042→//   - bins: Number of histogram bins
  1043→//
  1044→// Returns slice of MetricHistoPoints, one per metric.
  1045→//
  1046→// Limitations:
  1047→//   - Maximum 5000 jobs (returns nil if more jobs match)
  1048→//   - Requires metric backend availability
  1049→//   - Bins based on metric peak values from cluster configuration
  1050→//
  1051→// Algorithm:
  1052→//  1. Query first 5001 jobs to check count limit
  1053→//  2. Load metric averages for all jobs via metricdispatch
  1054→//  3. For each metric, create bins based on peak value
  1055→//  4. Iterate averages and count jobs per bin
  1056→func (r *JobRepository) runningJobsMetricStatisticsHistogram(
  1057→	ctx context.Context,
  1058→	metrics []string,
  1059→	filters []*model.JobFilter,
  1060→	bins *int,
  1061→) []*model.MetricHistoPoints {
  1062→	// Get Jobs
  1063→	jobs, err := r.QueryJobs(ctx, filters, &model.PageRequest{Page: 1, ItemsPerPage: 5000 + 1}, nil)
  1064→	if err != nil {
  1065→		cclog.Errorf("Error while querying jobs for footprint: %s", err)
  1066→		return nil
  1067→	}
  1068→	if len(jobs) > 5000 {
  1069→		cclog.Errorf("too many jobs matched (max: %d)", 5000)
  1070→		return nil
  1071→	}
  1072→
  1073→	// Get AVGs from metric repo
  1074→	avgs := make([][]schema.Float, len(metrics))
  1075→	for i := range avgs {
  1076→		avgs[i] = make([]schema.Float, 0, len(jobs))
  1077→	}
  1078→
  1079→	for _, job := range jobs {
  1080→		if job.MonitoringStatus == schema.MonitoringStatusDisabled || job.MonitoringStatus == schema.MonitoringStatusArchivingFailed {
  1081→			continue
  1082→		}
  1083→
  1084→		if err := metricdispatch.LoadAverages(job, metrics, avgs, ctx); err != nil {
  1085→			cclog.Errorf("Error while loading averages for histogram: %s", err)
  1086→			return nil
  1087→		}
  1088→	}
  1089→
  1090→	// Iterate metrics to fill endresult
  1091→	data := make([]*model.MetricHistoPoints, 0)
  1092→	for idx, metric := range metrics {
  1093→		// Get specific Peak or largest Peak
  1094→		var metricConfig *schema.MetricConfig
  1095→		var peak float64
  1096→		var unit string
  1097→
  1098→		for _, f := range filters {
  1099→			if f.Cluster != nil {
  1100→				metricConfig = archive.GetMetricConfig(*f.Cluster.Eq, metric)
  1101→				peak = metricConfig.Peak
  1102→				unit = metricConfig.Unit.Prefix + metricConfig.Unit.Base
  1103→			}
  1104→		}
  1105→
  1106→		if peak == 0.0 {
  1107→			for _, c := range archive.Clusters {
  1108→				for _, m := range c.MetricConfig {
  1109→					if m.Name == metric {
  1110→						if m.Peak > peak {
  1111→							peak = m.Peak
  1112→						}
  1113→						if unit == "" {
  1114→							unit = m.Unit.Prefix + m.Unit.Base
  1115→						}
  1116→					}
  1117→				}
  1118→			}
  1119→		}
  1120→
  1121→		// Make and fill bins
  1122→		peakBin := int(peak) / *bins
  1123→
  1124→		points := make([]*model.MetricHistoPoint, 0)
  1125→		for b := 0; b < *bins; b++ {
  1126→			count := 0
  1127→			bindex := b + 1
  1128→			bmin := peakBin * b
  1129→			bmax := peakBin * (b + 1)
  1130→
  1131→			// Iterate AVG values for indexed metric and count for bins
  1132→			for _, val := range avgs[idx] {
  1133→				if int(val) >= bmin && int(val) < bmax {
  1134→					count += 1
  1135→				}
  1136→			}
  1137→
  1138→			// Append Bin to Metric Result Array
  1139→			point := model.MetricHistoPoint{Bin: &bindex, Count: count, Min: &bmin, Max: &bmax}
  1140→			points = append(points, &point)
  1141→		}
  1142→
  1143→		// Append Metric Result Array to final results array
  1144→		result := model.MetricHistoPoints{Metric: metric, Unit: unit, Data: points}
  1145→		data = append(data, &result)
  1146→	}
  1147→
  1148→	return data
  1149→}
  1150→


   383→func MemoryUsageTracker(wg *sync.WaitGroup, ctx context.Context) {
   384→	ms := GetMemoryStore()
   385→
   386→	wg.Go(func() {
   387→		normalInterval := DefaultMemoryUsageTrackerInterval
   388→		fastInterval := 30 * time.Second
   389→
   390→		if normalInterval <= 0 {
   391→			return
   392→		}
   393→
   394→		ticker := time.NewTicker(normalInterval)
   395→		defer ticker.Stop()
   396→		currentInterval := normalInterval
   397→
   398→		for {
   399→			select {
   400→			case <-ctx.Done():
   401→				return
   402→			case <-ticker.C:
   403→				var mem runtime.MemStats
   404→				runtime.ReadMemStats(&mem)
   405→				actualMemoryGB := float64(mem.Alloc) / 1e9
   406→				metricDataGB := ms.SizeInGB()
   407→				cclog.Infof("[METRICSTORE]> memory usage: %.2f GB actual (%.2f GB metric data)", actualMemoryGB, metricDataGB)
   408→
   409→				freedExcluded := 0
   410→				freedEmergency := 0
   411→				var err error
   412→
   413→				state.mu.RLock()
   414→				lastRetention := state.lastRetentionTime
   415→				selectorsExcluded := state.selectorsExcluded
   416→				state.mu.RUnlock()
   417→
   418→				if lastRetention != 0 && selectorsExcluded {
   419→					freedExcluded, err = ms.Free(nil, lastRetention)
   420→					if err != nil {
   421→						cclog.Errorf("[METRICSTORE]> error while force-freeing the excluded buffers: %s", err)
   422→					}
   423→
   424→					if freedExcluded > 0 {
   425→						debug.FreeOSMemory()
   426→						cclog.Infof("[METRICSTORE]> done: %d excluded buffers force-freed", freedExcluded)
   427→					}
   428→				}
   429→
   430→				runtime.ReadMemStats(&mem)
   431→				actualMemoryGB = float64(mem.Alloc) / 1e9
   432→


grep -n "UpdateFootprint\|UpdateEnergy\|MarkArchived\|Execute" /Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/job.go | head -30