     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file contains job statistics and histogram generation functionality for the JobRepository.
     7→//
     8→// # Job Statistics
     9→//
    10→// The statistics methods provide aggregated metrics about jobs including total jobs, users,
    11→// walltime, and resource usage (nodes, cores, accelerators). Statistics can be computed:
    12→//   - Overall (JobsStats): Single aggregate across all matching jobs
    13→//   - Grouped (JobsStatsGrouped): Aggregated by user, project, cluster, or subcluster
    14→//   - Counts (JobCountGrouped, AddJobCount): Simple job counts with optional filtering
    15→//
    16→// All statistics methods support filtering via JobFilter and respect security contexts.
    17→//
    18→// # Histograms
    19→//
    20→// Histogram methods generate distribution data for visualization:
    21→//   - Duration, nodes, cores, accelerators (AddHistograms)
    22→//   - Job metrics like CPU load, memory usage (AddMetricHistograms)
    23→//
    24→// Histograms use intelligent binning:
    25→//   - Duration: Variable bin sizes (1m, 10m, 1h, 6h, 12h, 24h) with zero-padding
    26→//   - Resources: Natural value-based bins
    27→//   - Metrics: Normalized to peak values with configurable bin counts
    28→//
    29→// # Running vs. Completed Jobs
    30→//
    31→// Statistics handle running jobs specially:
    32→//   - Duration calculated as (now - start_time) for running jobs
    33→//   - Metric histograms for running jobs load data from metric backend instead of footprint
    34→//   - Job state filtering distinguishes running/completed jobs
    35→//
    36→// # Performance Considerations
    37→//
    38→// - All queries use prepared statements via stmtCache
    39→// - Complex aggregations use SQL for efficiency
    40→// - Histogram pre-initialization ensures consistent bin ranges
    41→// - Metric histogram queries limited to 5000 jobs for running job analysis
    42→
    43→package repository
    44→
    45→import (
    46→	"context"
    47→	"database/sql"
    48→	"fmt"
    49→	"time"
    50→
    51→	"github.com/ClusterCockpit/cc-backend/internal/config"
    52→	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
    53→	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
    54→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    55→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    56→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    57→	sq "github.com/Masterminds/squirrel"
    58→	"golang.org/x/sync/errgroup"
    59→)
    60→
    61→// groupBy2column maps GraphQL Aggregate enum values to their corresponding database column names.
    62→// Used by JobsStatsGrouped and JobCountGrouped to translate user-facing grouping dimensions
    63→// into SQL GROUP BY clauses. GraphQL validation ensures only valid enum values are accepted.
    64→var groupBy2column = map[model.Aggregate]string{
    65→	model.AggregateUser:       "job.hpc_user",
    66→	model.AggregateProject:    "job.project",
    67→	model.AggregateCluster:    "job.cluster",
    68→	model.AggregateSubcluster: "job.subcluster",
    69→}
    70→
    71→// sortBy2column maps GraphQL SortByAggregate enum values to their corresponding computed column names.
    72→// Used by JobsStatsGrouped to translate sort preferences into SQL ORDER BY clauses.
    73→// Column names match the AS aliases used in buildStatsQuery.
    74→var sortBy2column = map[model.SortByAggregate]string{
    75→	model.SortByAggregateTotaljobs:      "totalJobs",
    76→	model.SortByAggregateTotalusers:     "totalUsers",
    77→	model.SortByAggregateTotalwalltime:  "totalWalltime",
    78→	model.SortByAggregateTotalnodes:     "totalNodes",
    79→	model.SortByAggregateTotalnodehours: "totalNodeHours",
    80→	model.SortByAggregateTotalcores:     "totalCores",
    81→	model.SortByAggregateTotalcorehours: "totalCoreHours",
    82→	model.SortByAggregateTotalaccs:      "totalAccs",
    83→	model.SortByAggregateTotalacchours:  "totalAccHours",
    84→}
    85→
    86→// buildCountQuery constructs a SQL query to count jobs with optional grouping and filtering.
    87→//
    88→// Parameters:
    89→//   - filter: Job filters to apply (cluster, user, time range, etc.)
    90→//   - kind: Special filter - "running" for running jobs only, "short" for jobs under threshold
    91→//   - col: Column name to GROUP BY; empty string for total count without grouping
    92→//
    93→// Returns a SelectBuilder that produces either:
    94→//   - Single count: COUNT(job.id) when col is empty
    95→//   - Grouped counts: col, COUNT(job.id) when col is specified
    96→//
    97→// The kind parameter enables counting specific job categories:
    98→//   - "running": Only jobs with job_state = 'running'
    99→//   - "short": Only jobs with duration < ShortRunningJobsDuration config value
   100→//   - empty: All jobs matching filters
   101→func (r *JobRepository) buildCountQuery(
   102→	filter []*model.JobFilter,
   103→	kind string,
   104→	col string,
   105→) sq.SelectBuilder {
   106→	var query sq.SelectBuilder
   107→
   108→	if col != "" {
   109→		query = sq.Select(col, "COUNT(*)").From("job").GroupBy(col)
   110→	} else {
   111→		query = sq.Select("COUNT(*)").From("job")
   112→	}
   113→
   114→	switch kind {
   115→	case "running":
   116→		query = query.Where("job.job_state = 'running'")
   117→	case "short":
   118→		query = query.Where("job.duration < ?", config.Keys.ShortRunningJobsDuration)
   119→	}
   120→
   121→	for _, f := range filter {
   122→		query = BuildWhereClause(f, query)
   123→	}
   124→
   125→	return query
   126→}
   127→
   128→// buildStatsQuery constructs a SQL query to compute job statistics with optional grouping.
   129→// Only requested columns are computed; unrequested columns select 0 as placeholder.
   130→//
   131→// Parameters:
   132→//   - filter: Job filters to apply (cluster, user, time range, etc.)
   133→//   - col: Column name to GROUP BY; empty string for overall statistics without grouping
   134→//   - shortThreshold: Duration threshold in seconds for counting short-running jobs
   135→//   - reqFields: Set of requested field names; nil means compute all fields
   136→func (r *JobRepository) buildStatsQuery(
   137→	filter []*model.JobFilter,
   138→	col string,
   139→	shortThreshold int,
   140→	reqFields map[string]bool,
   141→) sq.SelectBuilder {
   142→	now := time.Now().Unix()
   143→
   144→	// Helper: return real expression if field is requested (or reqFields is nil), else "0 as alias"
   145→	need := func(field string) bool {
   146→		return reqFields == nil || reqFields[field]
   147→	}
   148→	durationExpr := fmt.Sprintf(`(CASE WHEN job.job_state = 'running' THEN %d - job.start_time ELSE job.duration END)`, now)
   149→
   150→	// Build column list
   151→	columns := make([]string, 0, 14)
   152→
   153→	if col != "" {
   154→		columns = append(columns, col)
   155→	}
   156→
   157→	columns = append(columns, "COUNT(*) as totalJobs")
   158→
   159→	if need("totalUsers") && col != "job.hpc_user" {
   160→		columns = append(columns, "COUNT(DISTINCT job.hpc_user) AS totalUsers")
   161→	} else {
   162→		columns = append(columns, "0 AS totalUsers")
   163→	}
   164→
   165→	if need("totalWalltime") {
   166→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s) / 3600) as int) as totalWalltime`, durationExpr))
   167→	} else {
   168→		columns = append(columns, "0 as totalWalltime")
   169→	}
   170→
   171→	if need("totalNodes") {
   172→		columns = append(columns, `CAST(SUM(job.num_nodes) as int) as totalNodes`)
   173→	} else {
   174→		columns = append(columns, "0 as totalNodes")
   175→	}
   176→
   177→	if need("totalNodeHours") {
   178→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s * job.num_nodes) / 3600) as int) as totalNodeHours`, durationExpr))
   179→	} else {
   180→		columns = append(columns, "0 as totalNodeHours")
   181→	}
   182→
   183→	if need("totalCores") {
   184→		columns = append(columns, `CAST(SUM(job.num_hwthreads) as int) as totalCores`)
   185→	} else {
   186→		columns = append(columns, "0 as totalCores")
   187→	}
   188→
   189→	if need("totalCoreHours") {
   190→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s * job.num_hwthreads) / 3600) as int) as totalCoreHours`, durationExpr))
   191→	} else {
   192→		columns = append(columns, "0 as totalCoreHours")
   193→	}
   194→
   195→	if need("totalAccs") {
   196→		columns = append(columns, `CAST(SUM(job.num_acc) as int) as totalAccs`)
   197→	} else {
   198→		columns = append(columns, "0 as totalAccs")
   199→	}
   200→
   201→	if need("totalAccHours") {
   202→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s * job.num_acc) / 3600) as int) as totalAccHours`, durationExpr))
   203→	} else {
   204→		columns = append(columns, "0 as totalAccHours")
   205→	}
   206→
   207→	if need("runningJobs") {
   208→		columns = append(columns, `COUNT(CASE WHEN job.job_state = 'running' THEN 1 END) as runningJobs`)
   209→	} else {
   210→		columns = append(columns, "0 as runningJobs")
   211→	}
   212→
   213→	if need("shortJobs") {
   214→		columns = append(columns, fmt.Sprintf(`COUNT(CASE WHEN job.duration < %d THEN 1 END) as shortJobs`, shortThreshold))
   215→	} else {
   216→		columns = append(columns, "0 as shortJobs")
   217→	}
   218→
   219→	query := sq.Select(columns...).From("job")
   220→	if col != "" {
   221→		query = query.GroupBy(col)
   222→	}
   223→
   224→	for _, f := range filter {
   225→		query = BuildWhereClause(f, query)
   226→	}
   227→
   228→	return query
   229→}
   230→
   231→// JobsStatsGrouped computes job statistics grouped by a dimension (user, project, cluster, or subcluster).
   232→// Only columns listed in reqFields are computed; others return 0. User display names are looked up
   233→// in a separate lightweight query to avoid JOIN overhead on the main aggregation.
   234→func (r *JobRepository) JobsStatsGrouped(
   235→	ctx context.Context,
   236→	filter []*model.JobFilter,
   237→	page *model.PageRequest,
   238→	sortBy *model.SortByAggregate,
   239→	groupBy *model.Aggregate,
   240→	reqFields map[string]bool,
   241→) ([]*model.JobsStatistics, error) {
   242→	col := groupBy2column[*groupBy]
   243→	query := r.buildStatsQuery(filter, col, config.Keys.ShortRunningJobsDuration, reqFields)
   244→
   245→	query, err := SecurityCheck(ctx, query)
   246→	if err != nil {
   247→		return nil, err
   248→	}
   249→
   250→	if sortBy != nil {
   251→		sortBy := sortBy2column[*sortBy]
   252→		query = query.OrderBy(fmt.Sprintf("%s DESC", sortBy))
   253→	}
   254→	if page != nil && page.ItemsPerPage != -1 {
   255→		limit := uint64(page.ItemsPerPage)
   256→		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
   257→	}
   258→
   259→	rows, err := query.RunWith(r.DB).QueryContext(ctx)
   260→	if err != nil {
   261→		cclog.Warn("Error while querying DB for job statistics")
   262→		return nil, err
   263→	}
   264→	defer rows.Close()
   265→
   266→	stats := make([]*model.JobsStatistics, 0, 100)
   267→
   268→	for rows.Next() {
   269→		var id sql.NullString
   270→		var jobs, users, walltime, nodes, nodeHours, cores, coreHours, accs, accHours, runningJobs, shortJobs sql.NullInt64
   271→		if err := rows.Scan(&id, &jobs, &users, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours, &runningJobs, &shortJobs); err != nil {
   272→			cclog.Warnf("Error while scanning rows: %s", err.Error())
   273→			return nil, err
   274→		}
   275→
   276→		if id.Valid {
   277→			stats = append(stats,
   278→				&model.JobsStatistics{
   279→					ID:             id.String,
   280→					TotalJobs:      int(jobs.Int64),
   281→					TotalUsers:     int(users.Int64),
   282→					TotalWalltime:  int(walltime.Int64),
   283→					TotalNodes:     int(nodes.Int64),
   284→					TotalNodeHours: int(nodeHours.Int64),
   285→					TotalCores:     int(cores.Int64),
   286→					TotalCoreHours: int(coreHours.Int64),
   287→					TotalAccs:      int(accs.Int64),
   288→					TotalAccHours:  int(accHours.Int64),
   289→					RunningJobs:    int(runningJobs.Int64),
   290→					ShortJobs:      int(shortJobs.Int64),
   291→				})
   292→		}
   293→	}
   294→
   295→	if err := rows.Err(); err != nil {
   296→		return nil, err
   297→	}
   298→
   299→	// Post-query name lookup for user grouping (avoids LEFT JOIN on aggregation query)
   300→	if col == "job.hpc_user" && len(stats) > 0 {
   301→		usernames := make([]any, len(stats))
   302→		for i, s := range stats {
   303→			usernames[i] = s.ID
   304→		}
   305→
   306→		nameQuery := sq.Select("username", "name").From("hpc_user").Where(sq.Eq{"username": usernames})
   307→		nameRows, err := nameQuery.RunWith(r.DB).QueryContext(ctx)
   308→		if err != nil {
   309→			cclog.Warnf("Error looking up user names: %s", err.Error())
   310→			// Non-fatal: stats are still valid without display names
   311→		} else {
   312→			defer nameRows.Close()
   313→			nameMap := make(map[string]string, len(stats))
   314→			for nameRows.Next() {
   315→				var username, name string
   316→				if err := nameRows.Scan(&username, &name); err == nil {
   317→					nameMap[username] = name
   318→				}
   319→			}
   320→			for _, s := range stats {
   321→				if name, ok := nameMap[s.ID]; ok {
   322→					s.Name = name
   323→				}
   324→			}
   325→		}
   326→	}
   327→
   328→	return stats, nil
   329→}
   330→
   331→// JobsStats computes overall job statistics across all matching jobs without grouping.
   332→//
   333→// This method provides a single aggregate view of job metrics, useful for dashboard
   334→// summaries and overall system utilization reports.
   335→//
   336→// Parameters:
   337→//   - ctx: Context for security checks and cancellation
   338→//   - filter: Filters to apply (time range, cluster, job state, etc.)
   339→//
   340→// Returns a single-element slice containing aggregate statistics:
   341→//   - totalJobs, totalUsers, totalWalltime
   342→//   - totalNodeHours, totalCoreHours, totalAccHours
   343→//
   344→// Unlike JobsStatsGrouped, this returns overall totals without breaking down by dimension.
   345→// Security checks are applied via SecurityCheck to respect user access levels.
   346→func (r *JobRepository) JobsStats(
   347→	ctx context.Context,
   348→	filter []*model.JobFilter,
   349→	reqFields map[string]bool,
   350→) ([]*model.JobsStatistics, error) {
   351→	start := time.Now()
   352→	query := r.buildStatsQuery(filter, "", config.Keys.ShortRunningJobsDuration, reqFields)
   353→	query, err := SecurityCheck(ctx, query)
   354→	if err != nil {
   355→		return nil, err
   356→	}
   357→
   358→	row := query.RunWith(r.DB).QueryRowContext(ctx)
   359→	stats := make([]*model.JobsStatistics, 0, 1)
   360→
   361→	var jobs, users, walltime, nodes, nodeHours, cores, coreHours, accs, accHours, runningJobs, shortJobs sql.NullInt64
   362→	if err := row.Scan(&jobs, &users, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours, &runningJobs, &shortJobs); err != nil {
   363→		cclog.Warn("Error while scanning rows")
   364→		return nil, err
   365→	}
   366→
   367→	if jobs.Valid {
   368→		var totalNodeHours, totalCoreHours, totalAccHours int
   369→
   370→		if nodeHours.Valid {
   371→			totalNodeHours = int(nodeHours.Int64)
   372→		}
   373→		if coreHours.Valid {
   374→			totalCoreHours = int(coreHours.Int64)
   375→		}
   376→		if accHours.Valid {
   377→			totalAccHours = int(accHours.Int64)
   378→		}
   379→		stats = append(stats,
   380→			&model.JobsStatistics{
   381→				TotalJobs:      int(jobs.Int64),
   382→				TotalUsers:     int(users.Int64),
   383→				TotalWalltime:  int(walltime.Int64),
   384→				TotalNodeHours: totalNodeHours,
   385→				TotalCoreHours: totalCoreHours,
   386→				TotalAccHours:  totalAccHours,
   387→				RunningJobs:    int(runningJobs.Int64),
   388→				ShortJobs:      int(shortJobs.Int64),
   389→			})
   390→	}
   391→
   392→	cclog.Debugf("Timer JobStats %s", time.Since(start))
   393→	return stats, nil
   394→}
   395→
   396→// LoadJobStat retrieves a specific statistic for a metric from a job's statistics.
   397→// Returns 0.0 if the metric is not found or statType is invalid.
   398→//
   399→// Parameters:
   400→//   - job: Job struct with populated Statistics field
   401→//   - metric: Name of the metric to query (e.g., "cpu_load", "mem_used")
   402→//   - statType: Type of statistic: "avg", "min", or "max"
   403→//
   404→// Returns the requested statistic value or 0.0 if not found.
   405→func LoadJobStat(job *schema.Job, metric string, statType string) float64 {
   406→	if stats, ok := job.Statistics[metric]; ok {
   407→		switch statType {
   408→		case "avg":
   409→			return stats.Avg
   410→		case "max":
   411→			return stats.Max
   412→		case "min":
   413→			return stats.Min
   414→		default:
   415→			cclog.Errorf("Unknown stat type %s", statType)
   416→		}
   417→	}
   418→
   419→	return 0.0
   420→}
   421→
   422→// JobCountGrouped counts jobs grouped by a dimension without computing detailed statistics.
   423→//
   424→// This is a lightweight alternative to JobsStatsGrouped when only job counts are needed,
   425→// avoiding the overhead of calculating walltime and resource usage metrics.
   426→//
   427→// Parameters:
   428→//   - ctx: Context for security checks
   429→//   - filter: Filters to apply
   430→//   - groupBy: Grouping dimension (User, Project, Cluster, or SubCluster)
   431→//
   432→// Returns JobsStatistics with only ID and TotalJobs populated for each group.
   433→func (r *JobRepository) JobCountGrouped(
   434→	ctx context.Context,
   435→	filter []*model.JobFilter,
   436→	groupBy *model.Aggregate,
   437→) ([]*model.JobsStatistics, error) {
   438→	start := time.Now()
   439→	col := groupBy2column[*groupBy]
   440→	query := r.buildCountQuery(filter, "", col)
   441→	query, err := SecurityCheck(ctx, query)
   442→	if err != nil {
   443→		return nil, err
   444→	}
   445→	rows, err := query.RunWith(r.DB).QueryContext(ctx)
   446→	if err != nil {
   447→		cclog.Warn("Error while querying DB for job statistics")
   448→		return nil, err
   449→	}
   450→	defer rows.Close()
   451→
   452→	stats := make([]*model.JobsStatistics, 0, 100)
   453→
   454→	for rows.Next() {
   455→		var id sql.NullString
   456→		var cnt sql.NullInt64
   457→		if err := rows.Scan(&id, &cnt); err != nil {
   458→			cclog.Warn("Error while scanning rows")
   459→			return nil, err
   460→		}
   461→		if id.Valid {
   462→			stats = append(stats,
   463→				&model.JobsStatistics{
   464→					ID:        id.String,
   465→					TotalJobs: int(cnt.Int64),
   466→				})
   467→		}
   468→	}
   469→
   470→	if err := rows.Err(); err != nil {
   471→		return nil, err
   472→	}
   473→
   474→	cclog.Debugf("Timer JobCountGrouped %s", time.Since(start))
   475→	return stats, nil
   476→}
   477→
   478→// AddJobCountGrouped augments existing statistics with additional job counts by category.
   479→//
   480→// This method enriches JobsStatistics returned by JobsStatsGrouped or JobCountGrouped
   481→// with counts of running or short-running (based on ShortRunningJobsDuration) jobs, matched by group ID.
   482→//
   483→// Parameters:
   484→//   - ctx: Context for security checks
   485→//   - filter: Filters to apply
   486→//   - groupBy: Grouping dimension (must match the dimension used for stats parameter)
   487→//   - stats: Existing statistics to augment (modified in-place by ID matching)
   488→//   - kind: "running" to add RunningJobs count, "short" to add ShortJobs count
   489→//
   490→// Returns the same stats slice with RunningJobs or ShortJobs fields populated per group.
   491→// Groups without matching jobs will have 0 for the added field.
   492→func (r *JobRepository) AddJobCountGrouped(
   493→	ctx context.Context,
   494→	filter []*model.JobFilter,
   495→	groupBy *model.Aggregate,
   496→	stats []*model.JobsStatistics,
   497→	kind string,
   498→) ([]*model.JobsStatistics, error) {
   499→	start := time.Now()
   500→	col := groupBy2column[*groupBy]
   501→	query := r.buildCountQuery(filter, kind, col)
   502→	query, err := SecurityCheck(ctx, query)
   503→	if err != nil {
   504→		return nil, err
   505→	}
   506→	rows, err := query.RunWith(r.DB).QueryContext(ctx)
   507→	if err != nil {
   508→		cclog.Warn("Error while querying DB for job statistics")
   509→		return nil, err
   510→	}
   511→	defer rows.Close()
   512→
   513→	counts := make(map[string]int)
   514→
   515→	for rows.Next() {
   516→		var id sql.NullString
   517→		var cnt sql.NullInt64
   518→		if err := rows.Scan(&id, &cnt); err != nil {
   519→			cclog.Warn("Error while scanning rows")
   520→			return nil, err
   521→		}
   522→		if id.Valid {
   523→			counts[id.String] = int(cnt.Int64)
   524→		}
   525→	}
   526→
   527→	if err := rows.Err(); err != nil {
   528→		return nil, err
   529→	}
   530→
   531→	switch kind {
   532→	case "running":
   533→		for _, s := range stats {
   534→			s.RunningJobs = counts[s.ID]
   535→		}
   536→	case "short":
   537→		for _, s := range stats {
   538→			s.ShortJobs = counts[s.ID]
   539→		}
   540→	}
   541→
   542→	cclog.Debugf("Timer AddJobCountGrouped %s", time.Since(start))
   543→	return stats, nil
   544→}
   545→
   546→// AddJobCount augments existing overall statistics with additional job counts by category.
   547→//
   548→// Similar to AddJobCountGrouped but for ungrouped statistics. Applies the same count
   549→// to all statistics entries (typically just one).
   550→//
   551→// Parameters:
   552→//   - ctx: Context for security checks
   553→//   - filter: Filters to apply
   554→//   - stats: Existing statistics to augment (modified in-place)
   555→//   - kind: "running" to add RunningJobs count, "short" to add ShortJobs count
   556→//
   557→// Returns the same stats slice with RunningJobs or ShortJobs fields set to the total count.
   558→func (r *JobRepository) AddJobCount(
   559→	ctx context.Context,
   560→	filter []*model.JobFilter,
   561→	stats []*model.JobsStatistics,
   562→	kind string,
   563→) ([]*model.JobsStatistics, error) {
   564→	start := time.Now()
   565→	query := r.buildCountQuery(filter, kind, "")
   566→	query, err := SecurityCheck(ctx, query)
   567→	if err != nil {
   568→		return nil, err
   569→	}
   570→	var cnt sql.NullInt64
   571→	if err := query.RunWith(r.DB).QueryRowContext(ctx).Scan(&cnt); err != nil {
   572→		cclog.Warn("Error while querying DB for job count")
   573→		return nil, err
   574→	}
   575→
   576→	count := int(cnt.Int64)
   577→
   578→	switch kind {
   579→	case "running":
   580→		for _, s := range stats {
   581→			s.RunningJobs = count
   582→		}
   583→	case "short":
   584→		for _, s := range stats {
   585→			s.ShortJobs = count
   586→		}
   587→	}
   588→
   589→	cclog.Debugf("Timer AddJobCount %s", time.Since(start))
   590→	return stats, nil
   591→}
   592→
   593→// AddHistograms augments statistics with distribution histograms for job properties.
   594→//
   595→// Generates histogram data for visualization of job duration, node count, core count,
   596→// and accelerator count distributions. Duration histogram uses intelligent binning based
   597→// on the requested resolution.
   598→//
   599→// Parameters:
   600→//   - ctx: Context for security checks
   601→//   - filter: Filters to apply to jobs included in histograms
   602→//   - stat: Statistics struct to augment (modified in-place)
   603→//   - durationBins: Bin size - "1m", "10m", "1h", "6h", "12h", or "24h" (default)
   604→//
   605→// Populates these fields in stat:
   606→//   - HistDuration: Job duration distribution (zero-padded bins)
   607→//   - HistNumNodes: Node count distribution
   608→//   - HistNumCores: Core (hwthread) count distribution
   609→//   - HistNumAccs: Accelerator count distribution
   610→//
   611→// Duration bins are pre-initialized with zeros to ensure consistent ranges for visualization.
   612→// Bin size determines both the width and maximum duration displayed (e.g., "1h" = 48 bins × 1h = 48h max).
   613→func (r *JobRepository) AddHistograms(
   614→	ctx context.Context,
   615→	filter []*model.JobFilter,
   616→	stat *model.JobsStatistics,
   617→	durationBins *string,
   618→) (*model.JobsStatistics, error) {
   619→	start := time.Now()
   620→
   621→	var targetBinCount int
   622→	var targetBinSize int
   623→	switch *durationBins {
   624→	case "1m": // 1 Minute Bins + Max 60 Bins -> Max 60 Minutes
   625→		targetBinCount = 60
   626→		targetBinSize = 60
   627→	case "10m": // 10 Minute Bins + Max 72 Bins -> Max 12 Hours
   628→		targetBinCount = 72
   629→		targetBinSize = 600
   630→	case "1h": // 1 Hour Bins + Max 48 Bins -> Max 48 Hours
   631→		targetBinCount = 48
   632→		targetBinSize = 3600
   633→	case "6h": // 6 Hour Bins + Max 12 Bins -> Max 3 Days
   634→		targetBinCount = 12
   635→		targetBinSize = 21600
   636→	case "12h": // 12 hour Bins + Max 14 Bins -> Max 7 Days
   637→		targetBinCount = 14
   638→		targetBinSize = 43200
   639→	default: // 24h
   640→		targetBinCount = 24
   641→		targetBinSize = 3600
   642→	}
   643→
   644→	// Run all 4 histogram queries in parallel — each writes a distinct struct field.
   645→	g, gctx := errgroup.WithContext(ctx)
   646→
   647→	value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = 'running' THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as int) as value`, time.Now().Unix(), targetBinSize)
   648→
   649→	g.Go(func() error {
   650→		var err error
   651→		stat.HistDuration, err = r.jobsDurationStatisticsHistogram(gctx, value, filter, targetBinSize, &targetBinCount)
   652→		if err != nil {
   653→			cclog.Warn("Error while loading job statistics histogram: job duration")
   654→		}
   655→		return err
   656→	})
   657→	g.Go(func() error {
   658→		var err error
   659→		stat.HistNumNodes, err = r.jobsStatisticsHistogram(gctx, "job.num_nodes as value", filter)
   660→		if err != nil {
   661→			cclog.Warn("Error while loading job statistics histogram: num nodes")
   662→		}
   663→		return err
   664→	})
   665→	g.Go(func() error {
   666→		var err error
   667→		stat.HistNumCores, err = r.jobsStatisticsHistogram(gctx, "job.num_hwthreads as value", filter)
   668→		if err != nil {
   669→			cclog.Warn("Error while loading job statistics histogram: num hwthreads")
   670→		}
   671→		return err
   672→	})
   673→	g.Go(func() error {
   674→		var err error
   675→		stat.HistNumAccs, err = r.jobsStatisticsHistogram(gctx, "job.num_acc as value", filter)
   676→		if err != nil {
   677→			cclog.Warn("Error while loading job statistics histogram: num acc")
   678→		}
   679→		return err
   680→	})
   681→
   682→	if err := g.Wait(); err != nil {
   683→		return nil, err
   684→	}
   685→
   686→	cclog.Debugf("Timer AddHistograms %s", time.Since(start))
   687→	return stat, nil
   688→}
   689→
   690→// AddMetricHistograms augments statistics with distribution histograms for job metrics.
   691→//
   692→// Generates histogram data for metrics like CPU load, memory usage, etc. Handles running
   693→// and completed jobs differently: running jobs load data from metric backend, completed jobs
   694→// use footprint data from database.
   695→//
   696→// Parameters:
   697→//   - ctx: Context for security checks
   698→//   - filter: Filters to apply (MUST contain State filter for running jobs)
   699→//   - metrics: List of metric names to histogram (e.g., ["cpu_load", "mem_used"])
   700→//   - stat: Statistics struct to augment (modified in-place)
   701→//   - targetBinCount: Number of histogram bins (default: 10)
   702→//
   703→// Populates HistMetrics field in stat with MetricHistoPoints for each metric.
   704→//
   705→// Binning algorithm:
   706→//   - Values normalized to metric's peak value from cluster configuration
   707→//   - Bins evenly distributed from 0 to peak
   708→//   - Pre-initialized with zeros for consistent visualization
   709→//
   710→// Limitations:
   711→//   - Running jobs: Limited to 5000 jobs for performance
   712→//   - Requires valid cluster configuration with metric peak values
   713→//   - Uses footprint statistic (avg/max/min) configured per metric
   714→func (r *JobRepository) AddMetricHistograms(
   715→	ctx context.Context,
   716→	filter []*model.JobFilter,
   717→	metrics []string,
   718→	stat *model.JobsStatistics,
   719→	targetBinCount *int,
   720→) (*model.JobsStatistics, error) {
   721→	start := time.Now()
   722→
   723→	// Running Jobs Only: First query jobdata from sqlite, then query data and make bins
   724→	for _, f := range filter {
   725→		if f.State != nil {
   726→			if len(f.State) == 1 && f.State[0] == "running" {
   727→				stat.HistMetrics = r.runningJobsMetricStatisticsHistogram(ctx, metrics, filter, targetBinCount)
   728→				cclog.Debugf("Timer AddMetricHistograms %s", time.Since(start))
   729→				return stat, nil
   730→			}
   731→		}
   732→	}
   733→
   734→	// All other cases: Query and make bins in sqlite directly
   735→	for _, m := range metrics {
   736→		metricHisto, err := r.jobsMetricStatisticsHistogram(ctx, m, filter, targetBinCount)
   737→		if err != nil {
   738→			cclog.Warnf("Error while loading job metric statistics histogram: %s", m)
   739→			continue
   740→		}
   741→		stat.HistMetrics = append(stat.HistMetrics, metricHisto)
   742→	}
   743→
   744→	cclog.Debugf("Timer AddMetricHistograms %s", time.Since(start))
   745→	return stat, nil
   746→}
   747→
   748→// jobsStatisticsHistogram generates a simple histogram by grouping on a column value.
   749→//
   750→// Used for histograms where the column value directly represents the bin (e.g., node count, core count).
   751→// Unlike duration/metric histograms, this doesn't pre-initialize bins with zeros.
   752→//
   753→// Parameters:
   754→//   - value: SQL expression that produces the histogram value, aliased as "value"
   755→//   - filters: Job filters to apply
   756→//
   757→// Returns histogram points with Value (from column) and Count (number of jobs).
   758→func (r *JobRepository) jobsStatisticsHistogram(
   759→	ctx context.Context,
   760→	value string,
   761→	filters []*model.JobFilter,
   762→) ([]*model.HistoPoint, error) {
   763→	start := time.Now()
   764→	query, qerr := SecurityCheck(ctx,
   765→		sq.Select(value, "COUNT(*) AS count").From("job"))
   766→
   767→	if qerr != nil {
   768→		return nil, qerr
   769→	}
   770→
   771→	for _, f := range filters {
   772→		query = BuildWhereClause(f, query)
   773→	}
   774→
   775→	rows, err := query.GroupBy("value").RunWith(r.DB).QueryContext(ctx)
   776→	if err != nil {
   777→		cclog.Error("Error while running query")
   778→		return nil, err
   779→	}
   780→	defer rows.Close()
   781→
   782→	points := make([]*model.HistoPoint, 0)
   783→	// is it possible to introduce zero values here? requires info about bincount
   784→	for rows.Next() {
   785→		point := model.HistoPoint{}
   786→		if err := rows.Scan(&point.Value, &point.Count); err != nil {
   787→			cclog.Warn("Error while scanning rows")
   788→			return nil, err
   789→		}
   790→
   791→		points = append(points, &point)
   792→	}
   793→
   794→	if err := rows.Err(); err != nil {
   795→		return nil, err
   796→	}
   797→
   798→	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
   799→	return points, nil
   800→}
   801→
   802→// jobsDurationStatisticsHistogram generates a duration histogram with pre-initialized bins.
   803→//
   804→// Bins are zero-padded to provide consistent ranges for visualization, unlike simple
   805→// histograms which only return bins with data. The value parameter should compute
   806→// the bin number from job duration.
   807→//
   808→// Parameters:
   809→//   - value: SQL expression computing bin number from duration, aliased as "value"
   810→//   - filters: Job filters to apply
   811→//   - binSizeSeconds: Width of each bin in seconds
   812→//   - targetBinCount: Number of bins to pre-initialize
   813→//
   814→// Returns histogram points with Value (bin_number × binSizeSeconds) and Count.
   815→// All bins from 1 to targetBinCount are returned, with Count=0 for empty bins.
   816→//
   817→// Algorithm:
   818→//  1. Pre-initialize targetBinCount bins with zero counts
   819→//  2. Query database for actual counts per bin
   820→//  3. Match query results to pre-initialized bins by value
   821→//  4. Bins without matches remain at zero
   822→func (r *JobRepository) jobsDurationStatisticsHistogram(
   823→	ctx context.Context,
   824→	value string,
   825→	filters []*model.JobFilter,
   826→	binSizeSeconds int,
   827→	targetBinCount *int,
   828→) ([]*model.HistoPoint, error) {
   829→	start := time.Now()
   830→	query, qerr := SecurityCheck(ctx,
   831→		sq.Select(value, "COUNT(*) AS count").From("job"))
   832→
   833→	if qerr != nil {
   834→		return nil, qerr
   835→	}
   836→
   837→	// Each bin represents a duration range: bin N = [N*binSizeSeconds, (N+1)*binSizeSeconds)
   838→	// Example: binSizeSeconds=3600 (1 hour), bin 1 = 0-1h, bin 2 = 1-2h, etc.
   839→	points := make([]*model.HistoPoint, 0, *targetBinCount)
   840→	for i := 1; i <= *targetBinCount; i++ {
   841→		point := model.HistoPoint{Value: i * binSizeSeconds, Count: 0}
   842→		points = append(points, &point)
   843→	}
   844→
   845→	// Build a map from bin value (seconds) to slice index for O(1) lookup.
   846→	binMap := make(map[int]int, len(points))
   847→	for i, p := range points {
   848→		binMap[p.Value] = i
   849→	}
   850→
   851→	for _, f := range filters {
   852→		query = BuildWhereClause(f, query)
   853→	}
   854→
   855→	rows, err := query.GroupBy("value").RunWith(r.DB).QueryContext(ctx)
   856→	if err != nil {
   857→		cclog.Error("Error while running query")
   858→		return nil, err
   859→	}
   860→	defer rows.Close()
   861→
   862→	// Match query results to pre-initialized bins using map lookup.
   863→	// point.Value from query is the bin number; multiply by binSizeSeconds to get bin key.
   864→	for rows.Next() {
   865→		point := model.HistoPoint{}
   866→		if err := rows.Scan(&point.Value, &point.Count); err != nil {
   867→			cclog.Warn("Error while scanning rows")
   868→			return nil, err
   869→		}
   870→
   871→		if idx, ok := binMap[point.Value*binSizeSeconds]; ok {
   872→			points[idx].Count = point.Count
   873→		}
   874→	}
   875→
   876→	if err := rows.Err(); err != nil {
   877→		return nil, err
   878→	}
   879→
   880→	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
   881→	return points, nil
   882→}
   883→
   884→// jobsMetricStatisticsHistogram generates a metric histogram using footprint data from completed jobs.
   885→//
   886→// Values are normalized to the metric's peak value and distributed into bins. The algorithm
   887→// is based on SQL histogram generation techniques, extracting metric values from JSON footprint
   888→// and computing bin assignments in SQL.
   889→//
   890→// Parameters:
   891→//   - metric: Metric name (e.g., "cpu_load", "mem_used")
   892→//   - filters: Job filters to apply
   893→//   - bins: Number of bins to generate
   894→//
   895→// Returns MetricHistoPoints with metric name, unit, footprint stat type, and binned data.
   896→//
   897→// Algorithm:
   898→//  1. Determine peak value from cluster configuration (filtered cluster or max across all)
   899→//  2. Generate SQL that extracts footprint value, normalizes to [0,1], multiplies by bin count
   900→//  3. Pre-initialize bins with min/max ranges based on peak value
   901→//  4. Query database for counts per bin
   902→//  5. Match results to pre-initialized bins
   903→//
   904→// Special handling: Values exactly equal to peak are forced into the last bin by multiplying
   905→// peak by 0.999999999 to avoid creating an extra bin.
   906→func (r *JobRepository) jobsMetricStatisticsHistogram(
   907→	ctx context.Context,
   908→	metric string,
   909→	filters []*model.JobFilter,
   910→	bins *int,
   911→) (*model.MetricHistoPoints, error) {
   912→	// Peak value defines the upper bound for binning: values are distributed across
   913→	// bins from 0 to peak. First try to get peak from filtered cluster, otherwise
   914→	// scan all clusters to find the maximum peak value.
   915→	var metricConfig *schema.MetricConfig
   916→	var peak float64
   917→	var unit string
   918→	var footprintStat string
   919→
   920→	// Try to get metric config from filtered cluster
   921→	for _, f := range filters {
   922→		if f.Cluster != nil {
   923→			metricConfig = archive.GetMetricConfig(*f.Cluster.Eq, metric)
   924→			peak = metricConfig.Peak
   925→			unit = metricConfig.Unit.Prefix + metricConfig.Unit.Base
   926→			footprintStat = metricConfig.Footprint
   927→			cclog.Debugf("Cluster %s filter found with peak %f for %s", *f.Cluster.Eq, peak, metric)
   928→		}
   929→	}
   930→
   931→	// If no cluster filter or peak not found, find largest peak across all clusters
   932→	// This ensures histogram can accommodate all possible values
   933→	if peak == 0.0 {
   934→		for _, c := range archive.Clusters {
   935→			for _, m := range c.MetricConfig {
   936→				if m.Name == metric {
   937→					if m.Peak > peak {
   938→						peak = m.Peak
   939→					}
   940→					if unit == "" {
   941→						unit = m.Unit.Prefix + m.Unit.Base
   942→					}
   943→					if footprintStat == "" {
   944→						footprintStat = m.Footprint
   945→					}
   946→				}
   947→			}
   948→		}
   949→	}
   950→
   951→	// Construct SQL histogram bins using normalized values.
   952→	// Algorithm based on: https://jereze.com/code/sql-histogram/ (modified)
   953→	start := time.Now()
   954→
   955→	// Bin calculation formula:
   956→	//   bin_number = CAST( (value / peak) * num_bins AS INTEGER ) + 1
   957→	// Special case: value == peak would create bin N+1, so we test for equality
   958→	// and multiply peak by 0.999999999 to force it into bin N.
   959→	binQuery := fmt.Sprintf(`CAST(
   960→		((case when json_extract(footprint, "$.%s") = %f then %f*0.999999999 else json_extract(footprint, "$.%s") end) / %f)
   961→		* %v as INTEGER )`,
   962→		(metric + "_" + footprintStat), peak, peak, (metric + "_" + footprintStat), peak, *bins)
   963→
   964→	mainQuery := sq.Select(
   965→		fmt.Sprintf(`%s + 1 as bin`, binQuery),
   966→		`count(*) as count`,
   967→	).From("job").Where(
   968→		"JSON_VALID(footprint)",
   969→	).Where(fmt.Sprintf(`json_extract(footprint, "$.%s") is not null and json_extract(footprint, "$.%s") <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak))
   970→
   971→	mainQuery, qerr := SecurityCheck(ctx, mainQuery)
   972→	if qerr != nil {
   973→		return nil, qerr
   974→	}
   975→
   976→	for _, f := range filters {
   977→		mainQuery = BuildWhereClause(f, mainQuery)
   978→	}
   979→
   980→	mainQuery = mainQuery.GroupBy("bin").OrderBy("bin")
   981→
   982→	rows, err := mainQuery.RunWith(r.DB).QueryContext(ctx)
   983→	if err != nil {
   984→		cclog.Errorf("Error while running mainQuery: %s", err)
   985→		return nil, err
   986→	}
   987→	defer rows.Close()
   988→
   989→	// Pre-initialize bins with calculated min/max ranges.
   990→	// Example: peak=1000, bins=10 -> bin 1=[0,100), bin 2=[100,200), ..., bin 10=[900,1000]
   991→	points := make([]*model.MetricHistoPoint, 0, *bins)
   992→	binStep := int(peak) / *bins
   993→	for i := 1; i <= *bins; i++ {
   994→		binMin := (binStep * (i - 1))
   995→		binMax := (binStep * i)
   996→		idx := i
   997→		epoint := model.MetricHistoPoint{Bin: &idx, Count: 0, Min: &binMin, Max: &binMax}
   998→		points = append(points, &epoint)
   999→	}
  1000→
  1001→	// Build a map from bin number to slice index for O(1) lookup.
  1002→	binMap := make(map[int]int, len(points))
  1003→	for i, p := range points {
  1004→		if p.Bin != nil {
  1005→			binMap[*p.Bin] = i
  1006→		}
  1007→	}
  1008→
  1009→	// Match query results to pre-initialized bins using map lookup.
  1010→	for rows.Next() {
  1011→		rpoint := model.MetricHistoPoint{}
  1012→		if err := rows.Scan(&rpoint.Bin, &rpoint.Count); err != nil {
  1013→			cclog.Warnf("Error while scanning rows for %s", metric)
  1014→			return nil, err
  1015→		}
  1016→
  1017→		if rpoint.Bin != nil {
  1018→			if idx, ok := binMap[*rpoint.Bin]; ok {
  1019→				points[idx].Count = rpoint.Count
  1020→			}
  1021→		}
  1022→	}
  1023→
  1024→	if err := rows.Err(); err != nil {
  1025→		return nil, err
  1026→	}
  1027→
  1028→	result := model.MetricHistoPoints{Metric: metric, Unit: unit, Stat: &footprintStat, Data: points}
  1029→
  1030→	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
  1031→	return &result, nil
  1032→}
  1033→
  1034→// runningJobsMetricStatisticsHistogram generates metric histograms for running jobs using live data.
  1035→//
  1036→// Unlike completed jobs which use footprint data from the database, running jobs require
  1037→// fetching current metric averages from the metric backend (via metricdispatch).
  1038→//
  1039→// Parameters:
  1040→//   - metrics: List of metric names
  1041→//   - filters: Job filters (should filter to running jobs only)
  1042→//   - bins: Number of histogram bins
  1043→//
  1044→// Returns slice of MetricHistoPoints, one per metric.
  1045→//
  1046→// Limitations:
  1047→//   - Maximum 5000 jobs (returns nil if more jobs match)
  1048→//   - Requires metric backend availability
  1049→//   - Bins based on metric peak values from cluster configuration
  1050→//
  1051→// Algorithm:
  1052→//  1. Query first 5001 jobs to check count limit
  1053→//  2. Load metric averages for all jobs via metricdispatch
  1054→//  3. For each metric, create bins based on peak value
  1055→//  4. Iterate averages and count jobs per bin
  1056→func (r *JobRepository) runningJobsMetricStatisticsHistogram(
  1057→	ctx context.Context,
  1058→	metrics []string,
  1059→	filters []*model.JobFilter,
  1060→	bins *int,
  1061→) []*model.MetricHistoPoints {
  1062→	// Get Jobs
  1063→	jobs, err := r.QueryJobs(ctx, filters, &model.PageRequest{Page: 1, ItemsPerPage: 5000 + 1}, nil)
  1064→	if err != nil {
  1065→		cclog.Errorf("Error while querying jobs for footprint: %s", err)
  1066→		return nil
  1067→	}
  1068→	if len(jobs) > 5000 {
  1069→		cclog.Errorf("too many jobs matched (max: %d)", 5000)
  1070→		return nil
  1071→	}
  1072→
  1073→	// Get AVGs from metric repo
  1074→	avgs := make([][]schema.Float, len(metrics))
  1075→	for i := range avgs {
  1076→		avgs[i] = make([]schema.Float, 0, len(jobs))
  1077→	}
  1078→
  1079→	for _, job := range jobs {
  1080→		if job.MonitoringStatus == schema.MonitoringStatusDisabled || job.MonitoringStatus == schema.MonitoringStatusArchivingFailed {
  1081→			continue
  1082→		}
  1083→
  1084→		if err := metricdispatch.LoadAverages(job, metrics, avgs, ctx); err != nil {
  1085→			cclog.Errorf("Error while loading averages for histogram: %s", err)
  1086→			return nil
  1087→		}
  1088→	}
  1089→
  1090→	// Iterate metrics to fill endresult
  1091→	data := make([]*model.MetricHistoPoints, 0)
  1092→	for idx, metric := range metrics {
  1093→		// Get specific Peak or largest Peak
  1094→		var metricConfig *schema.MetricConfig
  1095→		var peak float64
  1096→		var unit string
  1097→
  1098→		for _, f := range filters {
  1099→			if f.Cluster != nil {
  1100→				metricConfig = archive.GetMetricConfig(*f.Cluster.Eq, metric)
  1101→				peak = metricConfig.Peak
  1102→				unit = metricConfig.Unit.Prefix + metricConfig.Unit.Base
  1103→			}
  1104→		}
  1105→
  1106→		if peak == 0.0 {
  1107→			for _, c := range archive.Clusters {
  1108→				for _, m := range c.MetricConfig {
  1109→					if m.Name == metric {
  1110→						if m.Peak > peak {
  1111→							peak = m.Peak
  1112→						}
  1113→						if unit == "" {
  1114→							unit = m.Unit.Prefix + m.Unit.Base
  1115→						}
  1116→					}
  1117→				}
  1118→			}
  1119→		}
  1120→
  1121→		// Make and fill bins
  1122→		peakBin := int(peak) / *bins
  1123→
  1124→		points := make([]*model.MetricHistoPoint, 0)
  1125→		for b := 0; b < *bins; b++ {
  1126→			count := 0
  1127→			bindex := b + 1
  1128→			bmin := peakBin * b
  1129→			bmax := peakBin * (b + 1)
  1130→
  1131→			// Iterate AVG values for indexed metric and count for bins
  1132→			for _, val := range avgs[idx] {
  1133→				if int(val) >= bmin && int(val) < bmax {
  1134→					count += 1
  1135→				}
  1136→			}
  1137→
  1138→			// Append Bin to Metric Result Array
  1139→			point := model.MetricHistoPoint{Bin: &bindex, Count: count, Min: &bmin, Max: &bmax}
  1140→			points = append(points, &point)
  1141→		}
  1142→
  1143→		// Append Metric Result Array to final results array
  1144→		result := model.MetricHistoPoints{Metric: metric, Unit: unit, Data: points}
  1145→		data = append(data, &result)
  1146→	}
  1147→
  1148→	return data
  1149→}
  1150→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→package repository
     6→
     7→import (
     8→	"encoding/json"
     9→	"maps"
    10→	"sync"
    11→	"time"
    12→
    13→	"github.com/ClusterCockpit/cc-backend/internal/config"
    14→	"github.com/ClusterCockpit/cc-backend/web"
    15→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    16→	"github.com/ClusterCockpit/cc-lib/v2/lrucache"
    17→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    18→	"github.com/jmoiron/sqlx"
    19→)
    20→
    21→var (
    22→	userCfgRepoOnce     sync.Once
    23→	userCfgRepoInstance *UserCfgRepo
    24→)
    25→
    26→type UserCfgRepo struct {
    27→	DB         *sqlx.DB
    28→	Lookup     *sqlx.Stmt
    29→	uiDefaults map[string]any
    30→	cache      *lrucache.Cache
    31→	lock       sync.RWMutex
    32→}
    33→
    34→func GetUserCfgRepo() *UserCfgRepo {
    35→	userCfgRepoOnce.Do(func() {
    36→		db := GetConnection()
    37→
    38→		lookupConfigStmt, err := db.DB.Preparex(`SELECT confkey, value FROM configuration WHERE configuration.username = ?`)
    39→		if err != nil {
    40→			cclog.Fatalf("User Config: Call 'db.DB.Preparex()' failed.\nError: %s\n", err.Error())
    41→		}
    42→
    43→		userCfgRepoInstance = &UserCfgRepo{
    44→			DB:         db.DB,
    45→			Lookup:     lookupConfigStmt,
    46→			uiDefaults: web.UIDefaultsMap,
    47→			cache:      lrucache.New(1024),
    48→		}
    49→	})
    50→
    51→	return userCfgRepoInstance
    52→}
    53→
    54→// Return the personalised UI config for the currently authenticated
    55→// user or return the plain default config.
    56→func (uCfg *UserCfgRepo) GetUIConfig(user *schema.User) (map[string]any, error) {
    57→	if user == nil {
    58→		copy := make(map[string]any, len(uCfg.uiDefaults))
    59→		maps.Copy(copy, uCfg.uiDefaults)
    60→		return copy, nil
    61→	}
    62→
    63→	// Is the cache invalidated in case the options are changed?
    64→	data := uCfg.cache.Get(user.Username, func() (any, time.Duration, int) {
    65→		uiconfig := make(map[string]any, len(uCfg.uiDefaults))
    66→		maps.Copy(uiconfig, uCfg.uiDefaults)
    67→
    68→		rows, err := uCfg.Lookup.Query(user.Username)
    69→		if err != nil {
    70→			cclog.Warnf("Error while looking up user uiconfig for user '%v'", user.Username)
    71→			return err, 0, 0
    72→		}
    73→
    74→		size := 0
    75→		defer rows.Close()
    76→		for rows.Next() {
    77→			var key, rawval string
    78→			if err := rows.Scan(&key, &rawval); err != nil {
    79→				cclog.Warn("Error while scanning user uiconfig values")
    80→				return err, 0, 0
    81→			}
    82→
    83→			var val any
    84→			if err := json.Unmarshal([]byte(rawval), &val); err != nil {
    85→				cclog.Warn("Error while unmarshaling raw user uiconfig json")
    86→				return err, 0, 0
    87→			}
    88→
    89→			size += len(key)
    90→			size += len(rawval)
    91→			uiconfig[key] = val
    92→		}
    93→
    94→		// Add global ShortRunningJobsDuration setting as jobList_hideShortRunningJobs
    95→		uiconfig["jobList_hideShortRunningJobs"] = config.Keys.ShortRunningJobsDuration
    96→
    97→		return uiconfig, 24 * time.Hour, size
    98→	})
    99→	if err, ok := data.(error); ok {
   100→		cclog.Error("Error in returned dataset")
   101→		return nil, err
   102→	}
   103→
   104→	return data.(map[string]any), nil
   105→}
   106→
   107→// If the context does not have a user, update the global ui configuration
   108→// without persisting it!  If there is a (authenticated) user, update only his
   109→// configuration.
   110→func (uCfg *UserCfgRepo) UpdateConfig(
   111→	key, value string,
   112→	user *schema.User,
   113→) error {
   114→	if user == nil {
   115→		return nil
   116→	}
   117→
   118→	if _, err := uCfg.DB.Exec(`REPLACE INTO configuration (username, confkey, value) VALUES (?, ?, ?)`, user.Username, key, value); err != nil {
   119→		cclog.Warnf("Error while replacing user config in DB for user '%v'", user.Username)
   120→		return err
   121→	}
   122→
   123→	uCfg.cache.Del(user.Username)
   124→	return nil
   125→}
   126→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→package repository
     6→
     7→import (
     8→	"fmt"
     9→
    10→	"github.com/jmoiron/sqlx"
    11→)
    12→
    13→// Transaction wraps a database transaction for job-related operations.
    14→type Transaction struct {
    15→	tx *sqlx.Tx
    16→}
    17→
    18→// TransactionInit begins a new transaction.
    19→func (r *JobRepository) TransactionInit() (*Transaction, error) {
    20→	tx, err := r.DB.Beginx()
    21→	if err != nil {
    22→		return nil, fmt.Errorf("beginning transaction: %w", err)
    23→	}
    24→	return &Transaction{tx: tx}, nil
    25→}
    26→
    27→// Commit commits the transaction.
    28→// After calling Commit, the transaction should not be used again.
    29→func (t *Transaction) Commit() error {
    30→	if t.tx == nil {
    31→		return fmt.Errorf("transaction already committed or rolled back")
    32→	}
    33→	err := t.tx.Commit()
    34→	t.tx = nil // Mark as completed
    35→	if err != nil {
    36→		return fmt.Errorf("committing transaction: %w", err)
    37→	}
    38→	return nil
    39→}
    40→
    41→// Rollback rolls back the transaction.
    42→// It's safe to call Rollback on an already committed or rolled back transaction.
    43→func (t *Transaction) Rollback() error {
    44→	if t.tx == nil {
    45→		return nil // Already committed/rolled back
    46→	}
    47→	err := t.tx.Rollback()
    48→	t.tx = nil // Mark as completed
    49→	if err != nil {
    50→		return fmt.Errorf("rolling back transaction: %w", err)
    51→	}
    52→	return nil
    53→}
    54→
    55→// TransactionEnd commits the transaction.
    56→// Deprecated: Use Commit() instead.
    57→func (r *JobRepository) TransactionEnd(t *Transaction) error {
    58→	return t.Commit()
    59→}
    60→
    61→// TransactionAddNamed executes a named query within the transaction.
    62→func (r *JobRepository) TransactionAddNamed(
    63→	t *Transaction,
    64→	query string,
    65→	args ...any,
    66→) (int64, error) {
    67→	if t.tx == nil {
    68→		return 0, fmt.Errorf("transaction is nil or already completed")
    69→	}
    70→
    71→	res, err := t.tx.NamedExec(query, args)
    72→	if err != nil {
    73→		return 0, fmt.Errorf("named exec: %w", err)
    74→	}
    75→
    76→	id, err := res.LastInsertId()
    77→	if err != nil {
    78→		return 0, fmt.Errorf("getting last insert id: %w", err)
    79→	}
    80→
    81→	return id, nil
    82→}
    83→
    84→// TransactionAdd executes a query within the transaction.
    85→func (r *JobRepository) TransactionAdd(t *Transaction, query string, args ...any) (int64, error) {
    86→	if t.tx == nil {
    87→		return 0, fmt.Errorf("transaction is nil or already completed")
    88→	}
    89→
    90→	res, err := t.tx.Exec(query, args...)
    91→	if err != nil {
    92→		return 0, fmt.Errorf("exec: %w", err)
    93→	}
    94→
    95→	id, err := res.LastInsertId()
    96→	if err != nil {
    97→		return 0, fmt.Errorf("getting last insert id: %w", err)
    98→	}
    99→
   100→	return id, nil
   101→}
   102→


Bash

find /Users/jan/prg/ClusterCockpit/cc-backend -type f -name "*.go" | head -20


Found 49 files
internal/repository/node.go
pkg/archive/sqliteBackend.go
internal/graph/generated/generated.go
internal/graph/schema.resolvers.go
internal/graph/model/models_gen.go
internal/repository/job.go
cmd/cc-backend/server.go
pkg/metricstore/config.go
pkg/metricstore/walCheckpoint.go
pkg/metricstore/configSchema.go
pkg/metricstore/level.go
pkg/metricstore/checkpoint.go
pkg/metricstore/metricstore.go
pkg/metricstore/buffer.go
pkg/metricstore/lineprotocol.go
cmd/cc-backend/main.go
internal/repository/jobFind.go
internal/config/config.go
internal/config/schema.go
internal/tagger/tagger.go
tools/archive-manager/main.go
tools/archive-migration/transforms.go
pkg/metricstore/metricstore_test.go
pkg/archive/s3Backend.go
pkg/archive/sqliteBackend_test.go
pkg/metricstore/archive.go
pkg/archive/nodelist.go
pkg/archive/parquet/nodestate_writer.go
pkg/archive/parquet/writer.go
pkg/archive/fsBackend.go
internal/taskmanager/retentionService.go
internal/taskmanager/stopJobsExceedTime.go
internal/taskmanager/taskManager.go
internal/taskmanager/updateDurationService.go
internal/taskmanager/updateFootprintService.go
pkg/archive/archive.go
pkg/archive/ConfigSchema.go
internal/taskmanager/commitJobService.go
internal/taskmanager/compressionService.go
internal/taskmanager/ldapSyncService.go
internal/taskmanager/nodestateRetentionService.go
internal/repository/hooks_test.go
internal/auth/auth_test.go
internal/archiver/archiveWorker.go
internal/auth/auth.go
internal/api/nats_test.go
internal/api/api_test.go
cmd/cc-backend/init.go
internal/taskmanager/doc.go


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package taskmanager provides a background task scheduler for the cc-backend.
     7→// It manages various periodic tasks such as job archiving (retention),
     8→// database compression, LDAP synchronization, and statistic updates.
     9→//
    10→// The package uses the gocron library to schedule tasks. Configuration
    11→// for the tasks is provided via JSON configs passed to the Start function.
    12→package taskmanager
    13→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package taskmanager
     7→
     8→import (
     9→	"bytes"
    10→	"encoding/json"
    11→	"time"
    12→
    13→	"github.com/ClusterCockpit/cc-backend/internal/auth"
    14→	"github.com/ClusterCockpit/cc-backend/internal/config"
    15→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    16→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    17→	"github.com/go-co-op/gocron/v2"
    18→)
    19→
    20→const (
    21→	DefaultCompressOlderThan = 7
    22→)
    23→
    24→// Retention defines the configuration for job retention policies.
    25→type Retention struct {
    26→	Policy             string `json:"policy"`
    27→	Format             string `json:"format"`
    28→	Age                int    `json:"age"`
    29→	IncludeDB          bool   `json:"include-db"`
    30→	OmitTagged         string `json:"omit-tagged"`
    31→	TargetKind         string `json:"target-kind"`
    32→	TargetPath         string `json:"target-path"`
    33→	TargetEndpoint     string `json:"target-endpoint"`
    34→	TargetBucket       string `json:"target-bucket"`
    35→	TargetAccessKey    string `json:"target-access-key"`
    36→	TargetSecretKey    string `json:"target-secret-key"`
    37→	TargetRegion       string `json:"target-region"`
    38→	TargetUsePathStyle bool   `json:"target-use-path-style"`
    39→	MaxFileSizeMB      int    `json:"max-file-size-mb"`
    40→}
    41→
    42→// CronFrequency defines the execution intervals for various background workers.
    43→type CronFrequency struct {
    44→	// Duration Update Worker [Defaults to '2m']
    45→	CommitJobWorker string `json:"commit-job-worker"`
    46→	// Duration Update Worker [Defaults to '5m']
    47→	DurationWorker string `json:"duration-worker"`
    48→	// Metric-Footprint Update Worker [Defaults to '10m']
    49→	FootprintWorker string `json:"footprint-worker"`
    50→}
    51→
    52→var (
    53→	s       gocron.Scheduler
    54→	jobRepo *repository.JobRepository
    55→	// Keys holds the configured frequencies for cron jobs.
    56→	Keys CronFrequency
    57→)
    58→
    59→// parseDuration parses a duration string and handles errors by logging them.
    60→// It returns the duration and any error encountered.
    61→func parseDuration(s string) (time.Duration, error) {
    62→	interval, err := time.ParseDuration(s)
    63→	if err != nil {
    64→		cclog.Warnf("Could not parse duration for sync interval: %v",
    65→			s)
    66→		return 0, err
    67→	}
    68→
    69→	if interval == 0 {
    70→		cclog.Info("TaskManager: Sync interval is zero")
    71→	}
    72→
    73→	return interval, nil
    74→}
    75→
    76→func initArchiveServices(config json.RawMessage) {
    77→	var cfg struct {
    78→		Retention   Retention `json:"retention"`
    79→		Compression int       `json:"compression"`
    80→	}
    81→	cfg.Retention.IncludeDB = true
    82→
    83→	if err := json.Unmarshal(config, &cfg); err != nil {
    84→		cclog.Errorf("error while unmarshaling raw config json: %v", err)
    85→	}
    86→
    87→	switch cfg.Retention.Policy {
    88→	case "delete":
    89→		RegisterRetentionDeleteService(cfg.Retention)
    90→	case "copy":
    91→		RegisterRetentionCopyService(cfg.Retention)
    92→	case "move":
    93→		RegisterRetentionMoveService(cfg.Retention)
    94→	}
    95→
    96→	if cfg.Compression > 0 {
    97→		RegisterCompressionService(cfg.Compression)
    98→	} else {
    99→		RegisterCompressionService(DefaultCompressOlderThan)
   100→	}
   101→}
   102→
   103→// Start initializes the task manager, parses configurations, and registers background tasks.
   104→// It starts the gocron scheduler.
   105→func Start(cronCfg, archiveConfig json.RawMessage) {
   106→	var err error
   107→	jobRepo = repository.GetJobRepository()
   108→	s, err = gocron.NewScheduler()
   109→	if err != nil {
   110→		cclog.Abortf("Taskmanager Start: Could not create gocron scheduler.\nError: %s\n", err.Error())
   111→	}
   112→
   113→	if config.Keys.StopJobsExceedingWalltime > 0 {
   114→		RegisterStopJobsExceedTime()
   115→	}
   116→
   117→	dec := json.NewDecoder(bytes.NewReader(cronCfg))
   118→	dec.DisallowUnknownFields()
   119→	if err := dec.Decode(&Keys); err != nil {
   120→		cclog.Errorf("error while decoding cron config: %v", err)
   121→	}
   122→
   123→	if archiveConfig != nil {
   124→		initArchiveServices(archiveConfig)
   125→	} else {
   126→		// Always enable compression
   127→		RegisterCompressionService(DefaultCompressOlderThan)
   128→	}
   129→
   130→	lc := auth.Keys.LdapConfig
   131→
   132→	if lc != nil && lc.SyncInterval != "" {
   133→		RegisterLdapSyncService(lc.SyncInterval)
   134→	}
   135→
   136→	RegisterFootprintWorker()
   137→	RegisterUpdateDurationWorker()
   138→	RegisterCommitJobService()
   139→
   140→	if config.Keys.NodeStateRetention != nil && config.Keys.NodeStateRetention.Policy != "" {
   141→		initNodeStateRetention()
   142→	}
   143→
   144→	s.Start()
   145→}
   146→
   147→func initNodeStateRetention() {
   148→	cfg := config.Keys.NodeStateRetention
   149→	age := cfg.Age
   150→	if age <= 0 {
   151→		age = 24
   152→	}
   153→
   154→	switch cfg.Policy {
   155→	case "delete":
   156→		RegisterNodeStateRetentionDeleteService(age)
   157→	case "move":
   158→		RegisterNodeStateRetentionMoveService(cfg)
   159→	default:
   160→		cclog.Warnf("Unknown nodestate-retention policy: %s", cfg.Policy)
   161→	}
   162→}
   163→
   164→// Shutdown stops the task manager and its scheduler.
   165→func Shutdown() {
   166→	if s != nil {
   167→		s.Shutdown()
   168→	}
   169→}
   170→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// Package main provides the entry point for the ClusterCockpit backend server.
     7→// This file contains HTTP server setup, routing configuration, and
     8→// authentication middleware integration.
     9→package main
    10→
    11→import (
    12→	"context"
    13→	"crypto/tls"
    14→	"encoding/json"
    15→	"errors"
    16→	"fmt"
    17→	"net"
    18→	"net/http"
    19→	"os"
    20→	"strings"
    21→	"time"
    22→
    23→	"github.com/99designs/gqlgen/graphql"
    24→	"github.com/99designs/gqlgen/graphql/handler"
    25→	"github.com/99designs/gqlgen/graphql/handler/transport"
    26→	"github.com/99designs/gqlgen/graphql/playground"
    27→	"github.com/ClusterCockpit/cc-backend/internal/api"
    28→	"github.com/ClusterCockpit/cc-backend/internal/archiver"
    29→	"github.com/ClusterCockpit/cc-backend/internal/auth"
    30→	"github.com/ClusterCockpit/cc-backend/internal/config"
    31→	"github.com/ClusterCockpit/cc-backend/internal/graph"
    32→	"github.com/ClusterCockpit/cc-backend/internal/graph/generated"
    33→	"github.com/ClusterCockpit/cc-backend/internal/routerConfig"
    34→	"github.com/ClusterCockpit/cc-backend/pkg/metricstore"
    35→	"github.com/ClusterCockpit/cc-backend/web"
    36→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    37→	"github.com/ClusterCockpit/cc-lib/v2/nats"
    38→	"github.com/ClusterCockpit/cc-lib/v2/runtime"
    39→	"github.com/go-chi/chi/v5"
    40→	"github.com/go-chi/chi/v5/middleware"
    41→	"github.com/go-chi/cors"
    42→	httpSwagger "github.com/swaggo/http-swagger"
    43→)
    44→
    45→var buildInfo web.Build
    46→
    47→// Environment variable names
    48→const (
    49→	envDebug = "DEBUG"
    50→)
    51→
    52→// Server encapsulates the HTTP server state and dependencies
    53→type Server struct {
    54→	router        chi.Router
    55→	server        *http.Server
    56→	restAPIHandle *api.RestAPI
    57→	natsAPIHandle *api.NatsAPI
    58→}
    59→
    60→func onFailureResponse(rw http.ResponseWriter, r *http.Request, err error) {
    61→	rw.Header().Add("Content-Type", "application/json")
    62→	rw.WriteHeader(http.StatusUnauthorized)
    63→	json.NewEncoder(rw).Encode(map[string]string{
    64→		"status": http.StatusText(http.StatusUnauthorized),
    65→		"error":  err.Error(),
    66→	})
    67→}
    68→
    69→// NewServer creates and initializes a new Server instance
    70→func NewServer(version, commit, buildDate string) (*Server, error) {
    71→	buildInfo = web.Build{Version: version, Hash: commit, Buildtime: buildDate}
    72→
    73→	s := &Server{
    74→		router: chi.NewRouter(),
    75→	}
    76→
    77→	if err := s.init(); err != nil {
    78→		return nil, err
    79→	}
    80→
    81→	return s, nil
    82→}
    83→
    84→func (s *Server) init() error {
    85→	// Setup the http.Handler/Router used by the server
    86→	graph.Init()
    87→	resolver := graph.GetResolverInstance()
    88→	graphQLServer := handler.New(
    89→		generated.NewExecutableSchema(generated.Config{Resolvers: resolver}))
    90→
    91→	graphQLServer.AddTransport(transport.POST{})
    92→
    93→	// Inject a per-request stats cache so that grouped statistics queries
    94→	// sharing the same (filter, groupBy) pair are executed only once.
    95→	graphQLServer.AroundOperations(func(ctx context.Context, next graphql.OperationHandler) graphql.ResponseHandler {
    96→		return next(graph.WithStatsGroupCache(ctx))
    97→	})
    98→
    99→	if os.Getenv(envDebug) != "1" {
   100→		// Having this handler means that a error message is returned via GraphQL instead of the connection simply beeing closed.
   101→		// The problem with this is that then, no more stacktrace is printed to stderr.
   102→		graphQLServer.SetRecoverFunc(func(ctx context.Context, err any) error {
   103→			switch e := err.(type) {
   104→			case string:
   105→				return fmt.Errorf("MAIN > Panic: %s", e)
   106→			case error:
   107→				return fmt.Errorf("MAIN > Panic caused by: %s", e.Error())
   108→			}
   109→
   110→			return errors.New("MAIN > Internal server error (panic)")
   111→		})
   112→	}
   113→
   114→	authHandle := auth.GetAuthInstance()
   115→
   116→	// Middleware must be defined before routes in chi
   117→	s.router.Use(func(next http.Handler) http.Handler {
   118→		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
   119→			start := time.Now()
   120→			ww := middleware.NewWrapResponseWriter(rw, r.ProtoMajor)
   121→			next.ServeHTTP(ww, r)
   122→			cclog.Debugf("%s %s (%d, %.02fkb, %dms)",
   123→				r.Method, r.URL.RequestURI(),
   124→				ww.Status(), float32(ww.BytesWritten())/1024,
   125→				time.Since(start).Milliseconds())
   126→		})
   127→	})
   128→	s.router.Use(middleware.Compress(5))
   129→	s.router.Use(middleware.Recoverer)
   130→	s.router.Use(cors.Handler(cors.Options{
   131→		AllowCredentials: true,
   132→		AllowedHeaders:   []string{"X-Requested-With", "Content-Type", "Authorization", "Origin"},
   133→		AllowedMethods:   []string{"GET", "POST", "HEAD", "OPTIONS"},
   134→		AllowedOrigins:   []string{"*"},
   135→	}))
   136→
   137→	s.restAPIHandle = api.New()
   138→
   139→	info := map[string]any{}
   140→	info["hasOpenIDConnect"] = false
   141→
   142→	if auth.Keys.OpenIDConfig != nil {
   143→		openIDConnect := auth.NewOIDC(authHandle)
   144→		openIDConnect.RegisterEndpoints(s.router)
   145→		info["hasOpenIDConnect"] = true
   146→	}
   147→
   148→	s.router.Get("/login", func(rw http.ResponseWriter, r *http.Request) {
   149→		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
   150→		cclog.Debugf("##%v##", info)
   151→		web.RenderTemplate(rw, "login.tmpl", &web.Page{Title: "Login", Build: buildInfo, Infos: info})
   152→	})
   153→	s.router.HandleFunc("/imprint", func(rw http.ResponseWriter, r *http.Request) {
   154→		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
   155→		web.RenderTemplate(rw, "imprint.tmpl", &web.Page{Title: "Imprint", Build: buildInfo})
   156→	})
   157→	s.router.HandleFunc("/privacy", func(rw http.ResponseWriter, r *http.Request) {
   158→		rw.Header().Add("Content-Type", "text/html; charset=utf-8")
   159→		web.RenderTemplate(rw, "privacy.tmpl", &web.Page{Title: "Privacy", Build: buildInfo})
   160→	})
   161→
   162→	if !config.Keys.DisableAuthentication {
   163→		// Create login failure handler (used by both /login and /jwt-login)
   164→		loginFailureHandler := func(rw http.ResponseWriter, r *http.Request, err error) {
   165→			rw.Header().Add("Content-Type", "text/html; charset=utf-8")
   166→			rw.WriteHeader(http.StatusUnauthorized)
   167→			web.RenderTemplate(rw, "login.tmpl", &web.Page{
   168→				Title:   "Login failed - ClusterCockpit",
   169→				MsgType: "alert-warning",
   170→				Message: err.Error(),
   171→				Build:   buildInfo,
   172→				Infos:   info,
   173→			})
   174→		}
   175→
   176→		s.router.Post("/login", authHandle.Login(loginFailureHandler).ServeHTTP)
   177→		s.router.HandleFunc("/jwt-login", authHandle.Login(loginFailureHandler).ServeHTTP)
   178→
   179→		s.router.Post("/logout", authHandle.Logout(
   180→			http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
   181→				rw.Header().Add("Content-Type", "text/html; charset=utf-8")
   182→				rw.WriteHeader(http.StatusOK)
   183→				web.RenderTemplate(rw, "login.tmpl", &web.Page{
   184→					Title:   "Bye - ClusterCockpit",
   185→					MsgType: "alert-info",
   186→					Message: "Logout successful",
   187→					Build:   buildInfo,
   188→					Infos:   info,
   189→				})
   190→			})).ServeHTTP)
   191→	}
   192→
   193→	if flagDev {
   194→		s.router.Handle("/playground", playground.Handler("GraphQL playground", "/query"))
   195→		s.router.Get("/swagger/*", httpSwagger.Handler(
   196→			httpSwagger.URL("http://"+config.Keys.Addr+"/swagger/doc.json")))
   197→	}
   198→
   199→	// Secured routes (require authentication)
   200→	s.router.Group(func(secured chi.Router) {
   201→		if !config.Keys.DisableAuthentication {
   202→			secured.Use(func(next http.Handler) http.Handler {
   203→				return authHandle.Auth(
   204→					next,
   205→					func(rw http.ResponseWriter, r *http.Request, err error) {
   206→						rw.WriteHeader(http.StatusUnauthorized)
   207→						web.RenderTemplate(rw, "login.tmpl", &web.Page{
   208→							Title:    "Authentication failed - ClusterCockpit",
   209→							MsgType:  "alert-danger",
   210→							Message:  err.Error(),
   211→							Build:    buildInfo,
   212→							Infos:    info,
   213→							Redirect: r.RequestURI,
   214→						})
   215→					})
   216→			})
   217→		}
   218→
   219→		secured.Handle("/query", graphQLServer)
   220→
   221→		secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) {
   222→			routerConfig.HandleSearchBar(rw, r, buildInfo)
   223→		})
   224→
   225→		routerConfig.SetupRoutes(secured, buildInfo)
   226→	})
   227→
   228→	// API routes (JWT token auth)
   229→	s.router.Route("/api", func(apiRouter chi.Router) {
   230→		// Main API routes with API auth
   231→		apiRouter.Group(func(securedapi chi.Router) {
   232→			if !config.Keys.DisableAuthentication {
   233→				securedapi.Use(func(next http.Handler) http.Handler {
   234→					return authHandle.AuthAPI(next, onFailureResponse)
   235→				})
   236→			}
   237→			s.restAPIHandle.MountAPIRoutes(securedapi)
   238→		})
   239→
   240→		// Metric store API routes with separate auth
   241→		apiRouter.Group(func(metricstoreapi chi.Router) {
   242→			if !config.Keys.DisableAuthentication {
   243→				metricstoreapi.Use(func(next http.Handler) http.Handler {
   244→					return authHandle.AuthMetricStoreAPI(next, onFailureResponse)
   245→				})
   246→			}
   247→			s.restAPIHandle.MountMetricStoreAPIRoutes(metricstoreapi)
   248→		})
   249→	})
   250→
   251→	// User API routes
   252→	s.router.Route("/userapi", func(userapi chi.Router) {
   253→		if !config.Keys.DisableAuthentication {
   254→			userapi.Use(func(next http.Handler) http.Handler {
   255→				return authHandle.AuthUserAPI(next, onFailureResponse)
   256→			})
   257→		}
   258→		s.restAPIHandle.MountUserAPIRoutes(userapi)
   259→	})
   260→
   261→	// Config API routes (uses Group with full paths to avoid shadowing
   262→	// the /config page route that is registered in the secured group)
   263→	s.router.Group(func(configapi chi.Router) {
   264→		if !config.Keys.DisableAuthentication {
   265→			configapi.Use(func(next http.Handler) http.Handler {
   266→				return authHandle.AuthConfigAPI(next, onFailureResponse)
   267→			})
   268→		}
   269→		s.restAPIHandle.MountConfigAPIRoutes(configapi)
   270→	})
   271→
   272→	// Frontend API routes
   273→	s.router.Route("/frontend", func(frontendapi chi.Router) {
   274→		if !config.Keys.DisableAuthentication {
   275→			frontendapi.Use(func(next http.Handler) http.Handler {
   276→				return authHandle.AuthFrontendAPI(next, onFailureResponse)
   277→			})
   278→		}
   279→		s.restAPIHandle.MountFrontendAPIRoutes(frontendapi)
   280→	})
   281→
   282→	if config.Keys.APISubjects != nil {
   283→		s.natsAPIHandle = api.NewNatsAPI()
   284→		if err := s.natsAPIHandle.StartSubscriptions(); err != nil {
   285→			return fmt.Errorf("starting NATS subscriptions: %w", err)
   286→		}
   287→	}
   288→
   289→	// 404 handler for pages and API routes
   290→	notFoundHandler := func(rw http.ResponseWriter, r *http.Request) {
   291→		if strings.HasPrefix(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/userapi/") ||
   292→			strings.HasPrefix(r.URL.Path, "/frontend/") || strings.HasPrefix(r.URL.Path, "/config/") {
   293→			rw.Header().Set("Content-Type", "application/json")
   294→			rw.WriteHeader(http.StatusNotFound)
   295→			json.NewEncoder(rw).Encode(map[string]string{
   296→				"status": "Resource not found",
   297→				"error":  "the requested endpoint does not exist",
   298→			})
   299→			return
   300→		}
   301→		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
   302→		rw.WriteHeader(http.StatusNotFound)
   303→		web.RenderTemplate(rw, "404.tmpl", &web.Page{
   304→			Title: "Page Not Found",
   305→			Build: buildInfo,
   306→		})
   307→	}
   308→
   309→	// Set NotFound on the router so chi uses it for all unmatched routes,
   310→	// including those under subrouters like /api, /userapi, /frontend, etc.
   311→	s.router.NotFound(notFoundHandler)
   312→
   313→	if config.Keys.EmbedStaticFiles {
   314→		if i, err := os.Stat("./var/img"); err == nil {
   315→			if i.IsDir() {
   316→				cclog.Info("Use local directory for static images")
   317→				s.router.Handle("/img/*", http.StripPrefix("/img/", http.FileServer(http.Dir("./var/img"))))
   318→			}
   319→		}
   320→		fileServer := http.StripPrefix("/", web.ServeFiles())
   321→		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
   322→			if web.StaticFileExists(r.URL.Path) {
   323→				fileServer.ServeHTTP(rw, r)
   324→				return
   325→			}
   326→			notFoundHandler(rw, r)
   327→		}))
   328→	} else {
   329→		staticDir := http.Dir(config.Keys.StaticFiles)
   330→		fileServer := http.FileServer(staticDir)
   331→		s.router.Handle("/*", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
   332→			f, err := staticDir.Open(r.URL.Path)
   333→			if err == nil {
   334→				f.Close()
   335→				fileServer.ServeHTTP(rw, r)
   336→				return
   337→			}
   338→			notFoundHandler(rw, r)
   339→		}))
   340→	}
   341→
   342→	return nil
   343→}
   344→
   345→// Server timeout defaults (in seconds)
   346→const (
   347→	defaultReadTimeout  = 20
   348→	defaultWriteTimeout = 20
   349→)
   350→
   351→func (s *Server) Start(ctx context.Context) error {
   352→	// Use configurable timeouts with defaults
   353→	readTimeout := time.Duration(defaultReadTimeout) * time.Second
   354→	writeTimeout := time.Duration(defaultWriteTimeout) * time.Second
   355→
   356→	s.server = &http.Server{
   357→		ReadTimeout:  readTimeout,
   358→		WriteTimeout: writeTimeout,
   359→		Handler:      s.router,
   360→		Addr:         config.Keys.Addr,
   361→	}
   362→
   363→	// Start http or https server
   364→	listener, err := net.Listen("tcp", config.Keys.Addr)
   365→	if err != nil {
   366→		return fmt.Errorf("starting listener on '%s': %w", config.Keys.Addr, err)
   367→	}
   368→
   369→	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHTTPTo != "" {
   370→		go func() {
   371→			http.ListenAndServe(":80", http.RedirectHandler(config.Keys.RedirectHTTPTo, http.StatusMovedPermanently))
   372→		}()
   373→	}
   374→
   375→	if config.Keys.HTTPSCertFile != "" && config.Keys.HTTPSKeyFile != "" {
   376→		cert, err := tls.LoadX509KeyPair(
   377→			config.Keys.HTTPSCertFile, config.Keys.HTTPSKeyFile)
   378→		if err != nil {
   379→			return fmt.Errorf("loading X509 keypair (check 'https-cert-file' and 'https-key-file' in config.json): %w", err)
   380→		}
   381→		listener = tls.NewListener(listener, &tls.Config{
   382→			Certificates: []tls.Certificate{cert},
   383→			CipherSuites: []uint16{
   384→				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
   385→				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
   386→			},
   387→			MinVersion:               tls.VersionTLS12,
   388→			PreferServerCipherSuites: true,
   389→		})
   390→		cclog.Infof("HTTPS server listening at %s...", config.Keys.Addr)
   391→	} else {
   392→		cclog.Infof("HTTP server listening at %s...", config.Keys.Addr)
   393→	}
   394→	//
   395→	// Because this program will want to bind to a privileged port (like 80), the listener must
   396→	// be established first, then the user can be changed, and after that,
   397→	// the actual http server can be started.
   398→	if err := runtime.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
   399→		return fmt.Errorf("dropping privileges: %w", err)
   400→	}
   401→
   402→	// Handle context cancellation for graceful shutdown
   403→	go func() {
   404→		<-ctx.Done()
   405→		shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
   406→		defer cancel()
   407→		if err := s.server.Shutdown(shutdownCtx); err != nil {
   408→			cclog.Errorf("Server shutdown error: %v", err)
   409→		}
   410→	}()
   411→
   412→	if err = s.server.Serve(listener); err != nil && err != http.ErrServerClosed {
   413→		return fmt.Errorf("server failed: %w", err)
   414→	}
   415→	return nil
   416→}
   417→
   418→func (s *Server) Shutdown(ctx context.Context) {
   419→	// Create a shutdown context with timeout
   420→	shutdownCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
   421→	defer cancel()
   422→
   423→	nc := nats.GetClient()
   424→	if nc != nil {
   425→		nc.Close()
   426→	}
   427→
   428→	// First shut down the server gracefully (waiting for all ongoing requests)
   429→	if err := s.server.Shutdown(shutdownCtx); err != nil {
   430→		cclog.Errorf("Server shutdown error: %v", err)
   431→	}
   432→
   433→	// Archive all the metric store data
   434→	ms := metricstore.GetMemoryStore()
   435→
   436→	if ms != nil {
   437→		metricstore.Shutdown()
   438→	}
   439→
   440→	// Shutdown archiver with 10 second timeout for fast shutdown
   441→	if err := archiver.Shutdown(10 * time.Second); err != nil {
   442→		cclog.Warnf("Archiver shutdown: %v", err)
   443→	}
   444→}
   445→


Agent

Find tag and job update patterns

Prompt:
Explore the cc-backend codebase for DB write patterns specifically around:

1. **Tag operations** - AddTagOrCreate, adding tags to jobs. These are often called in loops (e.g., when starting a job with multiple tags). Check internal/repository/tags.go or similar files.

2. **Job archiving writes** - What happens when a job is archived? Does it update the job table? Check internal/archiver/ for the archiving flow and what DB writes it does.

3. **Job import/restore** - Check internal/importer/ for bulk import operations.

4. **Energy footprint / statistics updates** - Any post-processing that updates job rows after archiving.

5. **TransferCachedJobToMain** in internal/repository/jobCreate.go - this does INSERT + DELETE as separate operations. Could these be in a transaction?

For each pattern, identify if multiple individual writes could be combined into transactions or batch operations.

Also check internal/api/rest.go for the REST endpoint handlers (startJob, stopJob, etc.) to see their DB write patterns.


Explore the cc-backend codebase for DB write patterns specifically around:

1. **Tag operations** - AddTagOrCreate, adding tags to jobs. These are often called in loops (e.g., when starting a job with multiple tags). Check internal/repository/tags.go or similar files.

2. **Job archiving writes** - What happens when a job is archived? Does it update the job table? Check internal/archiver/ for the archiving flow and what DB writes it does.

3. **Job import/restore** - Check internal/importer/ for bulk import operations.

4. **Energy footprint / statistics updates** - Any post-processing that updates job rows after archiving.

5. **TransferCachedJobToMain** in internal/repository/jobCreate.go - this does INSERT + DELETE as separate operations. Could these be in a transaction?

For each pattern, identify if multiple individual writes could be combined into transactions or batch operations.

Also check internal/api/rest.go for the REST endpoint handlers (startJob, stopJob, etc.) to see their DB write patterns.

Prompt:
Explore the cc-backend codebase for DB write patterns specifically around:

1. **Tag operations** - AddTagOrCreate, adding tags to jobs. These are often called in loops (e.g., when starting a job with multiple tags). Check internal/repository/tags.go or similar files.

2. **Job archiving writes** - What happens when a job is archived? Does it update the job table? Check internal/archiver/ for the archiving flow and what DB writes it does.

3. **Job import/restore** - Check internal/importer/ for bulk import operations.

4. **Energy footprint / statistics updates** - Any post-processing that updates job rows after archiving.

5. **TransferCachedJobToMain** in internal/repository/jobCreate.go - this does INSERT + DELETE as separate operations. Could these be in a transaction?

For each pattern, identify if multiple individual writes could be combined into transactions or batch operations.

Also check internal/api/rest.go for the REST endpoint handlers (startJob, stopJob, etc.) to see their DB write patterns.


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package api
     7→
     8→import (
     9→	"bufio"
    10→	"database/sql"
    11→	"encoding/json"
    12→	"errors"
    13→	"fmt"
    14→	"net/http"
    15→	"strconv"
    16→	"strings"
    17→	"sync"
    18→	"time"
    19→
    20→	"github.com/ClusterCockpit/cc-backend/internal/archiver"
    21→	"github.com/ClusterCockpit/cc-backend/internal/config"
    22→	"github.com/ClusterCockpit/cc-backend/internal/graph"
    23→	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
    24→	"github.com/ClusterCockpit/cc-backend/internal/importer"
    25→	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
    26→	"github.com/ClusterCockpit/cc-backend/internal/repository"
    27→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    28→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    29→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    30→	"github.com/go-chi/chi/v5"
    31→)
    32→
    33→const (
    34→	// secondsPerDay is the number of seconds in 24 hours.
    35→	// Used for duplicate job detection within a day window.
    36→	secondsPerDay = 86400
    37→)
    38→
    39→// StopJobAPIRequest model
    40→type StopJobAPIRequest struct {
    41→	JobID     *int64          `json:"jobId" example:"123000"`
    42→	Cluster   *string         `json:"cluster" example:"fritz"`
    43→	StartTime *int64          `json:"startTime" example:"1649723812"`
    44→	State     schema.JobState `json:"jobState" validate:"required" example:"completed"`
    45→	StopTime  int64           `json:"stopTime" validate:"required" example:"1649763839"`
    46→}
    47→
    48→// DeleteJobAPIRequest model
    49→type DeleteJobAPIRequest struct {
    50→	JobID     *int64  `json:"jobId" validate:"required" example:"123000"` // Cluster Job ID of job
    51→	Cluster   *string `json:"cluster" example:"fritz"`                    // Cluster of job
    52→	StartTime *int64  `json:"startTime" example:"1649723812"`             // Start Time of job as epoch
    53→}
    54→
    55→// GetJobsAPIResponse model
    56→type GetJobsAPIResponse struct {
    57→	Jobs  []*schema.Job `json:"jobs"`  // Array of jobs
    58→	Items int           `json:"items"` // Number of jobs returned
    59→	Page  int           `json:"page"`  // Page id returned
    60→}
    61→
    62→// APITag model
    63→type APITag struct {
    64→	// Tag Type
    65→	Type  string `json:"type" example:"Debug"`
    66→	Name  string `json:"name" example:"Testjob"` // Tag Name
    67→	Scope string `json:"scope" example:"global"` // Tag Scope for Frontend Display
    68→}
    69→
    70→type EditMetaRequest struct {
    71→	Key   string `json:"key" example:"jobScript"`
    72→	Value string `json:"value" example:"bash script"`
    73→}
    74→
    75→// JobMetaRequest model
    76→type JobMetaRequest struct {
    77→	JobId     *int64          `json:"jobId" validate:"required" example:"123000"` // Cluster Job ID of job
    78→	Cluster   *string         `json:"cluster" example:"fritz"`                    // Cluster of job
    79→	StartTime *int64          `json:"startTime" example:"1649723812"`             // Start Time of job as epoch
    80→	Payload   EditMetaRequest `json:"payload"`                                    // Content to Add to Job Meta_Data
    81→}
    82→
    83→type TagJobAPIRequest []*APITag
    84→
    85→type GetJobAPIRequest []string
    86→
    87→type GetJobAPIResponse struct {
    88→	Meta *schema.Job
    89→	Data []*JobMetricWithName
    90→}
    91→
    92→type GetCompleteJobAPIResponse struct {
    93→	Meta *schema.Job
    94→	Data schema.JobData
    95→}
    96→
    97→type JobMetricWithName struct {
    98→	Metric *schema.JobMetric  `json:"metric"`
    99→	Name   string             `json:"name"`
   100→	Scope  schema.MetricScope `json:"scope"`
   101→}
   102→
   103→// getJobs godoc
   104→// @summary     Lists all jobs
   105→// @tags Job query
   106→// @description Get a list of all jobs. Filters can be applied using query parameters.
   107→// @description Number of results can be limited by page. Results are sorted by descending startTime.
   108→// @produce     json
   109→// @param       state          query    string            false "Job State" Enums(running, completed, failed, cancelled, stopped, timeout)
   110→// @param       cluster        query    string            false "Job Cluster"
   111→// @param       start-time     query    string            false "Syntax: '$from-$to', as unix epoch timestamps in seconds"
   112→// @param       items-per-page query    int               false "Items per page (Default: 25)"
   113→// @param       page           query    int               false "Page Number (Default: 1)"
   114→// @param       with-metadata  query    bool              false "Include metadata (e.g. jobScript) in response"
   115→// @success     200            {object} api.GetJobsAPIResponse  "Job array and page info"
   116→// @failure     400            {object} api.ErrorResponse       "Bad Request"
   117→// @failure     401   		   {object} api.ErrorResponse       "Unauthorized"
   118→// @failure     403            {object} api.ErrorResponse       "Forbidden"
   119→// @failure     500            {object} api.ErrorResponse       "Internal Server Error"
   120→// @security    ApiKeyAuth
   121→// @router      /api/jobs/ [get]
   122→func (api *RestAPI) getJobs(rw http.ResponseWriter, r *http.Request) {
   123→	withMetadata := false
   124→	filter := &model.JobFilter{}
   125→	page := &model.PageRequest{ItemsPerPage: 25, Page: 1}
   126→	order := &model.OrderByInput{Field: "startTime", Type: "col", Order: model.SortDirectionEnumDesc}
   127→
   128→	for key, vals := range r.URL.Query() {
   129→		switch key {
   130→		case "project":
   131→			filter.Project = &model.StringInput{Eq: &vals[0]}
   132→		case "state":
   133→			for _, s := range vals {
   134→				state := schema.JobState(s)
   135→				if !state.Valid() {
   136→					handleError(fmt.Errorf("invalid query parameter value: state"),
   137→						http.StatusBadRequest, rw)
   138→					return
   139→				}
   140→				filter.State = append(filter.State, state)
   141→			}
   142→		case "cluster":
   143→			filter.Cluster = &model.StringInput{Eq: &vals[0]}
   144→		case "start-time": // ?startTime=1753707480-1754053139
   145→			st := strings.Split(vals[0], "-")
   146→			if len(st) != 2 {
   147→				handleError(fmt.Errorf("invalid query parameter value: startTime"),
   148→					http.StatusBadRequest, rw)
   149→				return
   150→			}
   151→			from, err := strconv.ParseInt(st[0], 10, 64)
   152→			if err != nil {
   153→				handleError(err, http.StatusBadRequest, rw)
   154→				return
   155→			}
   156→			to, err := strconv.ParseInt(st[1], 10, 64)
   157→			if err != nil {
   158→				handleError(err, http.StatusBadRequest, rw)
   159→				return
   160→			}
   161→			ufrom, uto := time.Unix(from, 0), time.Unix(to, 0)
   162→			filter.StartTime = &config.TimeRange{From: &ufrom, To: &uto}
   163→		case "page":
   164→			x, err := strconv.Atoi(vals[0])
   165→			if err != nil {
   166→				handleError(err, http.StatusBadRequest, rw)
   167→				return
   168→			}
   169→			page.Page = x
   170→		case "items-per-page":
   171→			x, err := strconv.Atoi(vals[0])
   172→			if err != nil {
   173→				handleError(err, http.StatusBadRequest, rw)
   174→				return
   175→			}
   176→			page.ItemsPerPage = x
   177→		case "with-metadata":
   178→			withMetadata = true
   179→		default:
   180→			handleError(fmt.Errorf("invalid query parameter: %s", key),
   181→				http.StatusBadRequest, rw)
   182→			return
   183→		}
   184→	}
   185→
   186→	jobs, err := api.JobRepository.QueryJobs(r.Context(), []*model.JobFilter{filter}, page, order)
   187→	if err != nil {
   188→		handleError(err, http.StatusInternalServerError, rw)
   189→		return
   190→	}
   191→
   192→	results := make([]*schema.Job, 0, len(jobs))
   193→	for _, job := range jobs {
   194→		if withMetadata {
   195→			if _, err = api.JobRepository.FetchMetadata(job); err != nil {
   196→				handleError(err, http.StatusInternalServerError, rw)
   197→				return
   198→			}
   199→		}
   200→