     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→// This file contains job statistics and histogram generation functionality for the JobRepository.
     7→//
     8→// # Job Statistics
     9→//
    10→// The statistics methods provide aggregated metrics about jobs including total jobs, users,
    11→// walltime, and resource usage (nodes, cores, accelerators). Statistics can be computed:
    12→//   - Overall (JobsStats): Single aggregate across all matching jobs
    13→//   - Grouped (JobsStatsGrouped): Aggregated by user, project, cluster, or subcluster
    14→//   - Counts (JobCountGrouped, AddJobCount): Simple job counts with optional filtering
    15→//
    16→// All statistics methods support filtering via JobFilter and respect security contexts.
    17→//
    18→// # Histograms
    19→//
    20→// Histogram methods generate distribution data for visualization:
    21→//   - Duration, nodes, cores, accelerators (AddHistograms)
    22→//   - Job metrics like CPU load, memory usage (AddMetricHistograms)
    23→//
    24→// Histograms use intelligent binning:
    25→//   - Duration: Variable bin sizes (1m, 10m, 1h, 6h, 12h, 24h) with zero-padding
    26→//   - Resources: Natural value-based bins
    27→//   - Metrics: Normalized to peak values with configurable bin counts
    28→//
    29→// # Running vs. Completed Jobs
    30→//
    31→// Statistics handle running jobs specially:
    32→//   - Duration calculated as (now - start_time) for running jobs
    33→//   - Metric histograms for running jobs load data from metric backend instead of footprint
    34→//   - Job state filtering distinguishes running/completed jobs
    35→//
    36→// # Performance Considerations
    37→//
    38→// - All queries use prepared statements via stmtCache
    39→// - Complex aggregations use SQL for efficiency
    40→// - Histogram pre-initialization ensures consistent bin ranges
    41→// - Metric histogram queries limited to 5000 jobs for running job analysis
    42→
    43→package repository
    44→
    45→import (
    46→	"context"
    47→	"database/sql"
    48→	"fmt"
    49→	"time"
    50→
    51→	"github.com/ClusterCockpit/cc-backend/internal/config"
    52→	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
    53→	"github.com/ClusterCockpit/cc-backend/internal/metricdispatch"
    54→	"github.com/ClusterCockpit/cc-backend/pkg/archive"
    55→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    56→	"github.com/ClusterCockpit/cc-lib/v2/schema"
    57→	sq "github.com/Masterminds/squirrel"
    58→)
    59→
    60→// groupBy2column maps GraphQL Aggregate enum values to their corresponding database column names.
    61→// Used by JobsStatsGrouped and JobCountGrouped to translate user-facing grouping dimensions
    62→// into SQL GROUP BY clauses. GraphQL validation ensures only valid enum values are accepted.
    63→var groupBy2column = map[model.Aggregate]string{
    64→	model.AggregateUser:       "job.hpc_user",
    65→	model.AggregateProject:    "job.project",
    66→	model.AggregateCluster:    "job.cluster",
    67→	model.AggregateSubcluster: "job.subcluster",
    68→}
    69→
    70→// sortBy2column maps GraphQL SortByAggregate enum values to their corresponding computed column names.
    71→// Used by JobsStatsGrouped to translate sort preferences into SQL ORDER BY clauses.
    72→// Column names match the AS aliases used in buildStatsQuery.
    73→var sortBy2column = map[model.SortByAggregate]string{
    74→	model.SortByAggregateTotaljobs:      "totalJobs",
    75→	model.SortByAggregateTotalusers:     "totalUsers",
    76→	model.SortByAggregateTotalwalltime:  "totalWalltime",
    77→	model.SortByAggregateTotalnodes:     "totalNodes",
    78→	model.SortByAggregateTotalnodehours: "totalNodeHours",
    79→	model.SortByAggregateTotalcores:     "totalCores",
    80→	model.SortByAggregateTotalcorehours: "totalCoreHours",
    81→	model.SortByAggregateTotalaccs:      "totalAccs",
    82→	model.SortByAggregateTotalacchours:  "totalAccHours",
    83→}
    84→
    85→// buildCountQuery constructs a SQL query to count jobs with optional grouping and filtering.
    86→//
    87→// Parameters:
    88→//   - filter: Job filters to apply (cluster, user, time range, etc.)
    89→//   - kind: Special filter - "running" for running jobs only, "short" for jobs under threshold
    90→//   - col: Column name to GROUP BY; empty string for total count without grouping
    91→//
    92→// Returns a SelectBuilder that produces either:
    93→//   - Single count: COUNT(job.id) when col is empty
    94→//   - Grouped counts: col, COUNT(job.id) when col is specified
    95→//
    96→// The kind parameter enables counting specific job categories:
    97→//   - "running": Only jobs with job_state = 'running'
    98→//   - "short": Only jobs with duration < ShortRunningJobsDuration config value
    99→//   - empty: All jobs matching filters
   100→func (r *JobRepository) buildCountQuery(
   101→	filter []*model.JobFilter,
   102→	kind string,
   103→	col string,
   104→) sq.SelectBuilder {
   105→	var query sq.SelectBuilder
   106→
   107→	if col != "" {
   108→		query = sq.Select(col, "COUNT(job.id)").From("job").GroupBy(col)
   109→	} else {
   110→		query = sq.Select("COUNT(job.id)").From("job")
   111→	}
   112→
   113→	switch kind {
   114→	case "running":
   115→		query = query.Where("job.job_state = ?", "running")
   116→	case "short":
   117→		query = query.Where("job.duration < ?", config.Keys.ShortRunningJobsDuration)
   118→	}
   119→
   120→	for _, f := range filter {
   121→		query = BuildWhereClause(f, query)
   122→	}
   123→
   124→	return query
   125→}
   126→
   127→// buildStatsQuery constructs a SQL query to compute job statistics with optional grouping.
   128→// Only requested columns are computed; unrequested columns select 0 as placeholder.
   129→//
   130→// Parameters:
   131→//   - filter: Job filters to apply (cluster, user, time range, etc.)
   132→//   - col: Column name to GROUP BY; empty string for overall statistics without grouping
   133→//   - shortThreshold: Duration threshold in seconds for counting short-running jobs
   134→//   - reqFields: Set of requested field names; nil means compute all fields
   135→func (r *JobRepository) buildStatsQuery(
   136→	filter []*model.JobFilter,
   137→	col string,
   138→	shortThreshold int,
   139→	reqFields map[string]bool,
   140→) sq.SelectBuilder {
   141→	now := time.Now().Unix()
   142→
   143→	// Helper: return real expression if field is requested (or reqFields is nil), else "0 as alias"
   144→	need := func(field string) bool {
   145→		return reqFields == nil || reqFields[field]
   146→	}
   147→	durationExpr := fmt.Sprintf(`(CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)`, now)
   148→
   149→	// Build column list
   150→	columns := make([]string, 0, 14)
   151→
   152→	if col != "" {
   153→		columns = append(columns, col)
   154→	}
   155→
   156→	columns = append(columns, "COUNT(job.id) as totalJobs")
   157→
   158→	if need("totalUsers") && col != "job.hpc_user" {
   159→		columns = append(columns, "COUNT(DISTINCT job.hpc_user) AS totalUsers")
   160→	} else {
   161→		columns = append(columns, "0 AS totalUsers")
   162→	}
   163→
   164→	if need("totalWalltime") {
   165→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s) / 3600) as int) as totalWalltime`, durationExpr))
   166→	} else {
   167→		columns = append(columns, "0 as totalWalltime")
   168→	}
   169→
   170→	if need("totalNodes") {
   171→		columns = append(columns, `CAST(SUM(job.num_nodes) as int) as totalNodes`)
   172→	} else {
   173→		columns = append(columns, "0 as totalNodes")
   174→	}
   175→
   176→	if need("totalNodeHours") {
   177→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s * job.num_nodes) / 3600) as int) as totalNodeHours`, durationExpr))
   178→	} else {
   179→		columns = append(columns, "0 as totalNodeHours")
   180→	}
   181→
   182→	if need("totalCores") {
   183→		columns = append(columns, `CAST(SUM(job.num_hwthreads) as int) as totalCores`)
   184→	} else {
   185→		columns = append(columns, "0 as totalCores")
   186→	}
   187→
   188→	if need("totalCoreHours") {
   189→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s * job.num_hwthreads) / 3600) as int) as totalCoreHours`, durationExpr))
   190→	} else {
   191→		columns = append(columns, "0 as totalCoreHours")
   192→	}
   193→
   194→	if need("totalAccs") {
   195→		columns = append(columns, `CAST(SUM(job.num_acc) as int) as totalAccs`)
   196→	} else {
   197→		columns = append(columns, "0 as totalAccs")
   198→	}
   199→
   200→	if need("totalAccHours") {
   201→		columns = append(columns, fmt.Sprintf(`CAST(ROUND(SUM(%s * job.num_acc) / 3600) as int) as totalAccHours`, durationExpr))
   202→	} else {
   203→		columns = append(columns, "0 as totalAccHours")
   204→	}
   205→
   206→	if need("runningJobs") {
   207→		columns = append(columns, `COUNT(CASE WHEN job.job_state = 'running' THEN 1 END) as runningJobs`)
   208→	} else {
   209→		columns = append(columns, "0 as runningJobs")
   210→	}
   211→
   212→	if need("shortJobs") {
   213→		columns = append(columns, fmt.Sprintf(`COUNT(CASE WHEN job.duration < %d THEN 1 END) as shortJobs`, shortThreshold))
   214→	} else {
   215→		columns = append(columns, "0 as shortJobs")
   216→	}
   217→
   218→	query := sq.Select(columns...).From("job")
   219→	if col != "" {
   220→		query = query.GroupBy(col)
   221→	}
   222→
   223→	for _, f := range filter {
   224→		query = BuildWhereClause(f, query)
   225→	}
   226→
   227→	return query
   228→}
   229→
   230→// JobsStatsGrouped computes job statistics grouped by a dimension (user, project, cluster, or subcluster).
   231→// Only columns listed in reqFields are computed; others return 0. User display names are looked up
   232→// in a separate lightweight query to avoid JOIN overhead on the main aggregation.
   233→func (r *JobRepository) JobsStatsGrouped(
   234→	ctx context.Context,
   235→	filter []*model.JobFilter,
   236→	page *model.PageRequest,
   237→	sortBy *model.SortByAggregate,
   238→	groupBy *model.Aggregate,
   239→	reqFields map[string]bool,
   240→) ([]*model.JobsStatistics, error) {
   241→	start := time.Now()
   242→	col := groupBy2column[*groupBy]
   243→	query := r.buildStatsQuery(filter, col, config.Keys.ShortRunningJobsDuration, reqFields)
   244→
   245→	query, err := SecurityCheck(ctx, query)
   246→	if err != nil {
   247→		return nil, err
   248→	}
   249→
   250→	if sortBy != nil {
   251→		sortBy := sortBy2column[*sortBy]
   252→		query = query.OrderBy(fmt.Sprintf("%s DESC", sortBy))
   253→	}
   254→	if page != nil && page.ItemsPerPage != -1 {
   255→		limit := uint64(page.ItemsPerPage)
   256→		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
   257→	}
   258→
   259→	rows, err := query.RunWith(r.DB).QueryContext(ctx)
   260→	if err != nil {
   261→		cclog.Warn("Error while querying DB for job statistics")
   262→		return nil, err
   263→	}
   264→	defer rows.Close()
   265→
   266→	stats := make([]*model.JobsStatistics, 0, 100)
   267→
   268→	for rows.Next() {
   269→		var id sql.NullString
   270→		var jobs, users, walltime, nodes, nodeHours, cores, coreHours, accs, accHours, runningJobs, shortJobs sql.NullInt64
   271→		if err := rows.Scan(&id, &jobs, &users, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours, &runningJobs, &shortJobs); err != nil {
   272→			cclog.Warnf("Error while scanning rows: %s", err.Error())
   273→			return nil, err
   274→		}
   275→
   276→		if id.Valid {
   277→			stats = append(stats,
   278→				&model.JobsStatistics{
   279→					ID:             id.String,
   280→					TotalJobs:      int(jobs.Int64),
   281→					TotalUsers:     int(users.Int64),
   282→					TotalWalltime:  int(walltime.Int64),
   283→					TotalNodes:     int(nodes.Int64),
   284→					TotalNodeHours: int(nodeHours.Int64),
   285→					TotalCores:     int(cores.Int64),
   286→					TotalCoreHours: int(coreHours.Int64),
   287→					TotalAccs:      int(accs.Int64),
   288→					TotalAccHours:  int(accHours.Int64),
   289→					RunningJobs:    int(runningJobs.Int64),
   290→					ShortJobs:      int(shortJobs.Int64),
   291→				})
   292→		}
   293→	}
   294→
   295→	if err := rows.Err(); err != nil {
   296→		return nil, err
   297→	}
   298→
   299→	// Post-query name lookup for user grouping (avoids LEFT JOIN on aggregation query)
   300→	if col == "job.hpc_user" && len(stats) > 0 {
   301→		usernames := make([]any, len(stats))
   302→		for i, s := range stats {
   303→			usernames[i] = s.ID
   304→		}
   305→
   306→		nameQuery := sq.Select("username", "name").From("hpc_user").Where(sq.Eq{"username": usernames})
   307→		nameRows, err := nameQuery.RunWith(r.DB).QueryContext(ctx)
   308→		if err != nil {
   309→			cclog.Warnf("Error looking up user names: %s", err.Error())
   310→			// Non-fatal: stats are still valid without display names
   311→		} else {
   312→			defer nameRows.Close()
   313→			nameMap := make(map[string]string, len(stats))
   314→			for nameRows.Next() {
   315→				var username, name string
   316→				if err := nameRows.Scan(&username, &name); err == nil {
   317→					nameMap[username] = name
   318→				}
   319→			}
   320→			for _, s := range stats {
   321→				if name, ok := nameMap[s.ID]; ok {
   322→					s.Name = name
   323→				}
   324→			}
   325→		}
   326→	}
   327→
   328→	cclog.Debugf("Timer JobsStatsGrouped %s", time.Since(start))
   329→	return stats, nil
   330→}
   331→
   332→// JobsStats computes overall job statistics across all matching jobs without grouping.
   333→//
   334→// This method provides a single aggregate view of job metrics, useful for dashboard
   335→// summaries and overall system utilization reports.
   336→//
   337→// Parameters:
   338→//   - ctx: Context for security checks and cancellation
   339→//   - filter: Filters to apply (time range, cluster, job state, etc.)
   340→//
   341→// Returns a single-element slice containing aggregate statistics:
   342→//   - totalJobs, totalUsers, totalWalltime
   343→//   - totalNodeHours, totalCoreHours, totalAccHours
   344→//
   345→// Unlike JobsStatsGrouped, this returns overall totals without breaking down by dimension.
   346→// Security checks are applied via SecurityCheck to respect user access levels.
   347→func (r *JobRepository) JobsStats(
   348→	ctx context.Context,
   349→	filter []*model.JobFilter,
   350→	reqFields map[string]bool,
   351→) ([]*model.JobsStatistics, error) {
   352→	start := time.Now()
   353→	query := r.buildStatsQuery(filter, "", config.Keys.ShortRunningJobsDuration, reqFields)
   354→	query, err := SecurityCheck(ctx, query)
   355→	if err != nil {
   356→		return nil, err
   357→	}
   358→
   359→	row := query.RunWith(r.DB).QueryRowContext(ctx)
   360→	stats := make([]*model.JobsStatistics, 0, 1)
   361→
   362→	var jobs, users, walltime, nodes, nodeHours, cores, coreHours, accs, accHours, runningJobs, shortJobs sql.NullInt64
   363→	if err := row.Scan(&jobs, &users, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours, &runningJobs, &shortJobs); err != nil {
   364→		cclog.Warn("Error while scanning rows")
   365→		return nil, err
   366→	}
   367→
   368→	if jobs.Valid {
   369→		var totalNodeHours, totalCoreHours, totalAccHours int
   370→
   371→		if nodeHours.Valid {
   372→			totalNodeHours = int(nodeHours.Int64)
   373→		}
   374→		if coreHours.Valid {
   375→			totalCoreHours = int(coreHours.Int64)
   376→		}
   377→		if accHours.Valid {
   378→			totalAccHours = int(accHours.Int64)
   379→		}
   380→		stats = append(stats,
   381→			&model.JobsStatistics{
   382→				TotalJobs:      int(jobs.Int64),
   383→				TotalUsers:     int(users.Int64),
   384→				TotalWalltime:  int(walltime.Int64),
   385→				TotalNodeHours: totalNodeHours,
   386→				TotalCoreHours: totalCoreHours,
   387→				TotalAccHours:  totalAccHours,
   388→				RunningJobs:    int(runningJobs.Int64),
   389→				ShortJobs:      int(shortJobs.Int64),
   390→			})
   391→	}
   392→
   393→	cclog.Debugf("Timer JobStats %s", time.Since(start))
   394→	return stats, nil
   395→}
   396→
   397→// LoadJobStat retrieves a specific statistic for a metric from a job's statistics.
   398→// Returns 0.0 if the metric is not found or statType is invalid.
   399→//
   400→// Parameters:
   401→//   - job: Job struct with populated Statistics field
   402→//   - metric: Name of the metric to query (e.g., "cpu_load", "mem_used")
   403→//   - statType: Type of statistic: "avg", "min", or "max"
   404→//
   405→// Returns the requested statistic value or 0.0 if not found.
   406→func LoadJobStat(job *schema.Job, metric string, statType string) float64 {
   407→	if stats, ok := job.Statistics[metric]; ok {
   408→		switch statType {
   409→		case "avg":
   410→			return stats.Avg
   411→		case "max":
   412→			return stats.Max
   413→		case "min":
   414→			return stats.Min
   415→		default:
   416→			cclog.Errorf("Unknown stat type %s", statType)
   417→		}
   418→	}
   419→
   420→	return 0.0
   421→}
   422→
   423→// JobCountGrouped counts jobs grouped by a dimension without computing detailed statistics.
   424→//
   425→// This is a lightweight alternative to JobsStatsGrouped when only job counts are needed,
   426→// avoiding the overhead of calculating walltime and resource usage metrics.
   427→//
   428→// Parameters:
   429→//   - ctx: Context for security checks
   430→//   - filter: Filters to apply
   431→//   - groupBy: Grouping dimension (User, Project, Cluster, or SubCluster)
   432→//
   433→// Returns JobsStatistics with only ID and TotalJobs populated for each group.
   434→func (r *JobRepository) JobCountGrouped(
   435→	ctx context.Context,
   436→	filter []*model.JobFilter,
   437→	groupBy *model.Aggregate,
   438→) ([]*model.JobsStatistics, error) {
   439→	start := time.Now()
   440→	col := groupBy2column[*groupBy]
   441→	query := r.buildCountQuery(filter, "", col)
   442→	query, err := SecurityCheck(ctx, query)
   443→	if err != nil {
   444→		return nil, err
   445→	}
   446→	rows, err := query.RunWith(r.DB).QueryContext(ctx)
   447→	if err != nil {
   448→		cclog.Warn("Error while querying DB for job statistics")
   449→		return nil, err
   450→	}
   451→	defer rows.Close()
   452→
   453→	stats := make([]*model.JobsStatistics, 0, 100)
   454→
   455→	for rows.Next() {
   456→		var id sql.NullString
   457→		var cnt sql.NullInt64
   458→		if err := rows.Scan(&id, &cnt); err != nil {
   459→			cclog.Warn("Error while scanning rows")
   460→			return nil, err
   461→		}
   462→		if id.Valid {
   463→			stats = append(stats,
   464→				&model.JobsStatistics{
   465→					ID:        id.String,
   466→					TotalJobs: int(cnt.Int64),
   467→				})
   468→		}
   469→	}
   470→
   471→	if err := rows.Err(); err != nil {
   472→		return nil, err
   473→	}
   474→
   475→	cclog.Debugf("Timer JobCountGrouped %s", time.Since(start))
   476→	return stats, nil
   477→}
   478→
   479→// AddJobCountGrouped augments existing statistics with additional job counts by category.
   480→//
   481→// This method enriches JobsStatistics returned by JobsStatsGrouped or JobCountGrouped
   482→// with counts of running or short-running (based on ShortRunningJobsDuration) jobs, matched by group ID.
   483→//
   484→// Parameters:
   485→//   - ctx: Context for security checks
   486→//   - filter: Filters to apply
   487→//   - groupBy: Grouping dimension (must match the dimension used for stats parameter)
   488→//   - stats: Existing statistics to augment (modified in-place by ID matching)
   489→//   - kind: "running" to add RunningJobs count, "short" to add ShortJobs count
   490→//
   491→// Returns the same stats slice with RunningJobs or ShortJobs fields populated per group.
   492→// Groups without matching jobs will have 0 for the added field.
   493→func (r *JobRepository) AddJobCountGrouped(
   494→	ctx context.Context,
   495→	filter []*model.JobFilter,
   496→	groupBy *model.Aggregate,
   497→	stats []*model.JobsStatistics,
   498→	kind string,
   499→) ([]*model.JobsStatistics, error) {
   500→	start := time.Now()
   501→	col := groupBy2column[*groupBy]
   502→	query := r.buildCountQuery(filter, kind, col)
   503→	query, err := SecurityCheck(ctx, query)
   504→	if err != nil {
   505→		return nil, err
   506→	}
   507→	rows, err := query.RunWith(r.DB).QueryContext(ctx)
   508→	if err != nil {
   509→		cclog.Warn("Error while querying DB for job statistics")
   510→		return nil, err
   511→	}
   512→	defer rows.Close()
   513→
   514→	counts := make(map[string]int)
   515→
   516→	for rows.Next() {
   517→		var id sql.NullString
   518→		var cnt sql.NullInt64
   519→		if err := rows.Scan(&id, &cnt); err != nil {
   520→			cclog.Warn("Error while scanning rows")
   521→			return nil, err
   522→		}
   523→		if id.Valid {
   524→			counts[id.String] = int(cnt.Int64)
   525→		}
   526→	}
   527→
   528→	if err := rows.Err(); err != nil {
   529→		return nil, err
   530→	}
   531→
   532→	switch kind {
   533→	case "running":
   534→		for _, s := range stats {
   535→			s.RunningJobs = counts[s.ID]
   536→		}
   537→	case "short":
   538→		for _, s := range stats {
   539→			s.ShortJobs = counts[s.ID]
   540→		}
   541→	}
   542→
   543→	cclog.Debugf("Timer AddJobCountGrouped %s", time.Since(start))
   544→	return stats, nil
   545→}
   546→
   547→// AddJobCount augments existing overall statistics with additional job counts by category.
   548→//
   549→// Similar to AddJobCountGrouped but for ungrouped statistics. Applies the same count
   550→// to all statistics entries (typically just one).
   551→//
   552→// Parameters:
   553→//   - ctx: Context for security checks
   554→//   - filter: Filters to apply
   555→//   - stats: Existing statistics to augment (modified in-place)
   556→//   - kind: "running" to add RunningJobs count, "short" to add ShortJobs count
   557→//
   558→// Returns the same stats slice with RunningJobs or ShortJobs fields set to the total count.
   559→func (r *JobRepository) AddJobCount(
   560→	ctx context.Context,
   561→	filter []*model.JobFilter,
   562→	stats []*model.JobsStatistics,
   563→	kind string,
   564→) ([]*model.JobsStatistics, error) {
   565→	start := time.Now()
   566→	query := r.buildCountQuery(filter, kind, "")
   567→	query, err := SecurityCheck(ctx, query)
   568→	if err != nil {
   569→		return nil, err
   570→	}
   571→	var cnt sql.NullInt64
   572→	if err := query.RunWith(r.DB).QueryRowContext(ctx).Scan(&cnt); err != nil {
   573→		cclog.Warn("Error while querying DB for job count")
   574→		return nil, err
   575→	}
   576→
   577→	count := int(cnt.Int64)
   578→
   579→	switch kind {
   580→	case "running":
   581→		for _, s := range stats {
   582→			s.RunningJobs = count
   583→		}
   584→	case "short":
   585→		for _, s := range stats {
   586→			s.ShortJobs = count
   587→		}
   588→	}
   589→
   590→	cclog.Debugf("Timer AddJobCount %s", time.Since(start))
   591→	return stats, nil
   592→}
   593→
   594→// AddHistograms augments statistics with distribution histograms for job properties.
   595→//
   596→// Generates histogram data for visualization of job duration, node count, core count,
   597→// and accelerator count distributions. Duration histogram uses intelligent binning based
   598→// on the requested resolution.
   599→//
   600→// Parameters:
   601→//   - ctx: Context for security checks
   602→//   - filter: Filters to apply to jobs included in histograms
   603→//   - stat: Statistics struct to augment (modified in-place)
   604→//   - durationBins: Bin size - "1m", "10m", "1h", "6h", "12h", or "24h" (default)
   605→//
   606→// Populates these fields in stat:
   607→//   - HistDuration: Job duration distribution (zero-padded bins)
   608→//   - HistNumNodes: Node count distribution
   609→//   - HistNumCores: Core (hwthread) count distribution
   610→//   - HistNumAccs: Accelerator count distribution
   611→//
   612→// Duration bins are pre-initialized with zeros to ensure consistent ranges for visualization.
   613→// Bin size determines both the width and maximum duration displayed (e.g., "1h" = 48 bins × 1h = 48h max).
   614→func (r *JobRepository) AddHistograms(
   615→	ctx context.Context,
   616→	filter []*model.JobFilter,
   617→	stat *model.JobsStatistics,
   618→	durationBins *string,
   619→) (*model.JobsStatistics, error) {
   620→	start := time.Now()
   621→
   622→	var targetBinCount int
   623→	var targetBinSize int
   624→	switch *durationBins {
   625→	case "1m": // 1 Minute Bins + Max 60 Bins -> Max 60 Minutes
   626→		targetBinCount = 60
   627→		targetBinSize = 60
   628→	case "10m": // 10 Minute Bins + Max 72 Bins -> Max 12 Hours
   629→		targetBinCount = 72
   630→		targetBinSize = 600
   631→	case "1h": // 1 Hour Bins + Max 48 Bins -> Max 48 Hours
   632→		targetBinCount = 48
   633→		targetBinSize = 3600
   634→	case "6h": // 6 Hour Bins + Max 12 Bins -> Max 3 Days
   635→		targetBinCount = 12
   636→		targetBinSize = 21600
   637→	case "12h": // 12 hour Bins + Max 14 Bins -> Max 7 Days
   638→		targetBinCount = 14
   639→		targetBinSize = 43200
   640→	default: // 24h
   641→		targetBinCount = 24
   642→		targetBinSize = 3600
   643→	}
   644→
   645→	var err error
   646→	// Return X-Values always as seconds, will be formatted into minutes and hours in frontend
   647→	value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as int) as value`, time.Now().Unix(), targetBinSize)
   648→	stat.HistDuration, err = r.jobsDurationStatisticsHistogram(ctx, value, filter, targetBinSize, &targetBinCount)
   649→	if err != nil {
   650→		cclog.Warn("Error while loading job statistics histogram: job duration")
   651→		return nil, err
   652→	}
   653→
   654→	stat.HistNumNodes, err = r.jobsStatisticsHistogram(ctx, "job.num_nodes as value", filter)
   655→	if err != nil {
   656→		cclog.Warn("Error while loading job statistics histogram: num nodes")
   657→		return nil, err
   658→	}
   659→
   660→	stat.HistNumCores, err = r.jobsStatisticsHistogram(ctx, "job.num_hwthreads as value", filter)
   661→	if err != nil {
   662→		cclog.Warn("Error while loading job statistics histogram: num hwthreads")
   663→		return nil, err
   664→	}
   665→
   666→	stat.HistNumAccs, err = r.jobsStatisticsHistogram(ctx, "job.num_acc as value", filter)
   667→	if err != nil {
   668→		cclog.Warn("Error while loading job statistics histogram: num acc")
   669→		return nil, err
   670→	}
   671→
   672→	cclog.Debugf("Timer AddHistograms %s", time.Since(start))
   673→	return stat, nil
   674→}
   675→
   676→// AddMetricHistograms augments statistics with distribution histograms for job metrics.
   677→//
   678→// Generates histogram data for metrics like CPU load, memory usage, etc. Handles running
   679→// and completed jobs differently: running jobs load data from metric backend, completed jobs
   680→// use footprint data from database.
   681→//
   682→// Parameters:
   683→//   - ctx: Context for security checks
   684→//   - filter: Filters to apply (MUST contain State filter for running jobs)
   685→//   - metrics: List of metric names to histogram (e.g., ["cpu_load", "mem_used"])
   686→//   - stat: Statistics struct to augment (modified in-place)
   687→//   - targetBinCount: Number of histogram bins (default: 10)
   688→//
   689→// Populates HistMetrics field in stat with MetricHistoPoints for each metric.
   690→//
   691→// Binning algorithm:
   692→//   - Values normalized to metric's peak value from cluster configuration
   693→//   - Bins evenly distributed from 0 to peak
   694→//   - Pre-initialized with zeros for consistent visualization
   695→//
   696→// Limitations:
   697→//   - Running jobs: Limited to 5000 jobs for performance
   698→//   - Requires valid cluster configuration with metric peak values
   699→//   - Uses footprint statistic (avg/max/min) configured per metric
   700→func (r *JobRepository) AddMetricHistograms(
   701→	ctx context.Context,
   702→	filter []*model.JobFilter,
   703→	metrics []string,
   704→	stat *model.JobsStatistics,
   705→	targetBinCount *int,
   706→) (*model.JobsStatistics, error) {
   707→	start := time.Now()
   708→
   709→	// Running Jobs Only: First query jobdata from sqlite, then query data and make bins
   710→	for _, f := range filter {
   711→		if f.State != nil {
   712→			if len(f.State) == 1 && f.State[0] == "running" {
   713→				stat.HistMetrics = r.runningJobsMetricStatisticsHistogram(ctx, metrics, filter, targetBinCount)
   714→				cclog.Debugf("Timer AddMetricHistograms %s", time.Since(start))
   715→				return stat, nil
   716→			}
   717→		}
   718→	}
   719→
   720→	// All other cases: Query and make bins in sqlite directly
   721→	for _, m := range metrics {
   722→		metricHisto, err := r.jobsMetricStatisticsHistogram(ctx, m, filter, targetBinCount)
   723→		if err != nil {
   724→			cclog.Warnf("Error while loading job metric statistics histogram: %s", m)
   725→			continue
   726→		}
   727→		stat.HistMetrics = append(stat.HistMetrics, metricHisto)
   728→	}
   729→
   730→	cclog.Debugf("Timer AddMetricHistograms %s", time.Since(start))
   731→	return stat, nil
   732→}
   733→
   734→// jobsStatisticsHistogram generates a simple histogram by grouping on a column value.
   735→//
   736→// Used for histograms where the column value directly represents the bin (e.g., node count, core count).
   737→// Unlike duration/metric histograms, this doesn't pre-initialize bins with zeros.
   738→//
   739→// Parameters:
   740→//   - value: SQL expression that produces the histogram value, aliased as "value"
   741→//   - filters: Job filters to apply
   742→//
   743→// Returns histogram points with Value (from column) and Count (number of jobs).
   744→func (r *JobRepository) jobsStatisticsHistogram(
   745→	ctx context.Context,
   746→	value string,
   747→	filters []*model.JobFilter,
   748→) ([]*model.HistoPoint, error) {
   749→	start := time.Now()
   750→	query, qerr := SecurityCheck(ctx,
   751→		sq.Select(value, "COUNT(job.id) AS count").From("job"))
   752→
   753→	if qerr != nil {
   754→		return nil, qerr
   755→	}
   756→
   757→	for _, f := range filters {
   758→		query = BuildWhereClause(f, query)
   759→	}
   760→
   761→	rows, err := query.GroupBy("value").RunWith(r.DB).QueryContext(ctx)
   762→	if err != nil {
   763→		cclog.Error("Error while running query")
   764→		return nil, err
   765→	}
   766→	defer rows.Close()
   767→
   768→	points := make([]*model.HistoPoint, 0)
   769→	// is it possible to introduce zero values here? requires info about bincount
   770→	for rows.Next() {
   771→		point := model.HistoPoint{}
   772→		if err := rows.Scan(&point.Value, &point.Count); err != nil {
   773→			cclog.Warn("Error while scanning rows")
   774→			return nil, err
   775→		}
   776→
   777→		points = append(points, &point)
   778→	}
   779→
   780→	if err := rows.Err(); err != nil {
   781→		return nil, err
   782→	}
   783→
   784→	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
   785→	return points, nil
   786→}
   787→
   788→// jobsDurationStatisticsHistogram generates a duration histogram with pre-initialized bins.
   789→//
   790→// Bins are zero-padded to provide consistent ranges for visualization, unlike simple
   791→// histograms which only return bins with data. The value parameter should compute
   792→// the bin number from job duration.
   793→//
   794→// Parameters:
   795→//   - value: SQL expression computing bin number from duration, aliased as "value"
   796→//   - filters: Job filters to apply
   797→//   - binSizeSeconds: Width of each bin in seconds
   798→//   - targetBinCount: Number of bins to pre-initialize
   799→//
   800→// Returns histogram points with Value (bin_number × binSizeSeconds) and Count.
   801→// All bins from 1 to targetBinCount are returned, with Count=0 for empty bins.
   802→//
   803→// Algorithm:
   804→//  1. Pre-initialize targetBinCount bins with zero counts
   805→//  2. Query database for actual counts per bin
   806→//  3. Match query results to pre-initialized bins by value
   807→//  4. Bins without matches remain at zero
   808→func (r *JobRepository) jobsDurationStatisticsHistogram(
   809→	ctx context.Context,
   810→	value string,
   811→	filters []*model.JobFilter,
   812→	binSizeSeconds int,
   813→	targetBinCount *int,
   814→) ([]*model.HistoPoint, error) {
   815→	start := time.Now()
   816→	query, qerr := SecurityCheck(ctx,
   817→		sq.Select(value, "COUNT(job.id) AS count").From("job"))
   818→
   819→	if qerr != nil {
   820→		return nil, qerr
   821→	}
   822→
   823→	// Each bin represents a duration range: bin N = [N*binSizeSeconds, (N+1)*binSizeSeconds)
   824→	// Example: binSizeSeconds=3600 (1 hour), bin 1 = 0-1h, bin 2 = 1-2h, etc.
   825→	points := make([]*model.HistoPoint, 0, *targetBinCount)
   826→	for i := 1; i <= *targetBinCount; i++ {
   827→		point := model.HistoPoint{Value: i * binSizeSeconds, Count: 0}
   828→		points = append(points, &point)
   829→	}
   830→
   831→	// Build a map from bin value (seconds) to slice index for O(1) lookup.
   832→	binMap := make(map[int]int, len(points))
   833→	for i, p := range points {
   834→		binMap[p.Value] = i
   835→	}
   836→
   837→	for _, f := range filters {
   838→		query = BuildWhereClause(f, query)
   839→	}
   840→
   841→	rows, err := query.GroupBy("value").RunWith(r.DB).QueryContext(ctx)
   842→	if err != nil {
   843→		cclog.Error("Error while running query")
   844→		return nil, err
   845→	}
   846→	defer rows.Close()
   847→
   848→	// Match query results to pre-initialized bins using map lookup.
   849→	// point.Value from query is the bin number; multiply by binSizeSeconds to get bin key.
   850→	for rows.Next() {
   851→		point := model.HistoPoint{}
   852→		if err := rows.Scan(&point.Value, &point.Count); err != nil {
   853→			cclog.Warn("Error while scanning rows")
   854→			return nil, err
   855→		}
   856→
   857→		if idx, ok := binMap[point.Value*binSizeSeconds]; ok {
   858→			points[idx].Count = point.Count
   859→		}
   860→	}
   861→
   862→	if err := rows.Err(); err != nil {
   863→		return nil, err
   864→	}
   865→
   866→	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
   867→	return points, nil
   868→}
   869→
   870→// jobsMetricStatisticsHistogram generates a metric histogram using footprint data from completed jobs.
   871→//
   872→// Values are normalized to the metric's peak value and distributed into bins. The algorithm
   873→// is based on SQL histogram generation techniques, extracting metric values from JSON footprint
   874→// and computing bin assignments in SQL.
   875→//
   876→// Parameters:
   877→//   - metric: Metric name (e.g., "cpu_load", "mem_used")
   878→//   - filters: Job filters to apply
   879→//   - bins: Number of bins to generate
   880→//
   881→// Returns MetricHistoPoints with metric name, unit, footprint stat type, and binned data.
   882→//
   883→// Algorithm:
   884→//  1. Determine peak value from cluster configuration (filtered cluster or max across all)
   885→//  2. Generate SQL that extracts footprint value, normalizes to [0,1], multiplies by bin count
   886→//  3. Pre-initialize bins with min/max ranges based on peak value
   887→//  4. Query database for counts per bin
   888→//  5. Match results to pre-initialized bins
   889→//
   890→// Special handling: Values exactly equal to peak are forced into the last bin by multiplying
   891→// peak by 0.999999999 to avoid creating an extra bin.
   892→func (r *JobRepository) jobsMetricStatisticsHistogram(
   893→	ctx context.Context,
   894→	metric string,
   895→	filters []*model.JobFilter,
   896→	bins *int,
   897→) (*model.MetricHistoPoints, error) {
   898→	// Peak value defines the upper bound for binning: values are distributed across
   899→	// bins from 0 to peak. First try to get peak from filtered cluster, otherwise
   900→	// scan all clusters to find the maximum peak value.
   901→	var metricConfig *schema.MetricConfig
   902→	var peak float64
   903→	var unit string
   904→	var footprintStat string
   905→
   906→	// Try to get metric config from filtered cluster
   907→	for _, f := range filters {
   908→		if f.Cluster != nil {
   909→			metricConfig = archive.GetMetricConfig(*f.Cluster.Eq, metric)
   910→			peak = metricConfig.Peak
   911→			unit = metricConfig.Unit.Prefix + metricConfig.Unit.Base
   912→			footprintStat = metricConfig.Footprint
   913→			cclog.Debugf("Cluster %s filter found with peak %f for %s", *f.Cluster.Eq, peak, metric)
   914→		}
   915→	}
   916→
   917→	// If no cluster filter or peak not found, find largest peak across all clusters
   918→	// This ensures histogram can accommodate all possible values
   919→	if peak == 0.0 {
   920→		for _, c := range archive.Clusters {
   921→			for _, m := range c.MetricConfig {
   922→				if m.Name == metric {
   923→					if m.Peak > peak {
   924→						peak = m.Peak
   925→					}
   926→					if unit == "" {
   927→						unit = m.Unit.Prefix + m.Unit.Base
   928→					}
   929→					if footprintStat == "" {
   930→						footprintStat = m.Footprint
   931→					}
   932→				}
   933→			}
   934→		}
   935→	}
   936→
   937→	// Construct SQL histogram bins using normalized values.
   938→	// Algorithm based on: https://jereze.com/code/sql-histogram/ (modified)
   939→	start := time.Now()
   940→
   941→	// Bin calculation formula:
   942→	//   bin_number = CAST( (value / peak) * num_bins AS INTEGER ) + 1
   943→	// Special case: value == peak would create bin N+1, so we test for equality
   944→	// and multiply peak by 0.999999999 to force it into bin N.
   945→	binQuery := fmt.Sprintf(`CAST(
   946→		((case when json_extract(footprint, "$.%s") = %f then %f*0.999999999 else json_extract(footprint, "$.%s") end) / %f)
   947→		* %v as INTEGER )`,
   948→		(metric + "_" + footprintStat), peak, peak, (metric + "_" + footprintStat), peak, *bins)
   949→
   950→	mainQuery := sq.Select(
   951→		fmt.Sprintf(`%s + 1 as bin`, binQuery),
   952→		`count(*) as count`,
   953→	).From("job").Where(
   954→		"JSON_VALID(footprint)",
   955→	).Where(fmt.Sprintf(`json_extract(footprint, "$.%s") is not null and json_extract(footprint, "$.%s") <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak))
   956→
   957→	mainQuery, qerr := SecurityCheck(ctx, mainQuery)
   958→	if qerr != nil {
   959→		return nil, qerr
   960→	}
   961→
   962→	for _, f := range filters {
   963→		mainQuery = BuildWhereClause(f, mainQuery)
   964→	}
   965→
   966→	mainQuery = mainQuery.GroupBy("bin").OrderBy("bin")
   967→
   968→	rows, err := mainQuery.RunWith(r.DB).QueryContext(ctx)
   969→	if err != nil {
   970→		cclog.Errorf("Error while running mainQuery: %s", err)
   971→		return nil, err
   972→	}
   973→	defer rows.Close()
   974→
   975→	// Pre-initialize bins with calculated min/max ranges.
   976→	// Example: peak=1000, bins=10 -> bin 1=[0,100), bin 2=[100,200), ..., bin 10=[900,1000]
   977→	points := make([]*model.MetricHistoPoint, 0, *bins)
   978→	binStep := int(peak) / *bins
   979→	for i := 1; i <= *bins; i++ {
   980→		binMin := (binStep * (i - 1))
   981→		binMax := (binStep * i)
   982→		idx := i
   983→		epoint := model.MetricHistoPoint{Bin: &idx, Count: 0, Min: &binMin, Max: &binMax}
   984→		points = append(points, &epoint)
   985→	}
   986→
   987→	// Build a map from bin number to slice index for O(1) lookup.
   988→	binMap := make(map[int]int, len(points))
   989→	for i, p := range points {
   990→		if p.Bin != nil {
   991→			binMap[*p.Bin] = i
   992→		}
   993→	}
   994→
   995→	// Match query results to pre-initialized bins using map lookup.
   996→	for rows.Next() {
   997→		rpoint := model.MetricHistoPoint{}
   998→		if err := rows.Scan(&rpoint.Bin, &rpoint.Count); err != nil {
   999→			cclog.Warnf("Error while scanning rows for %s", metric)
  1000→			return nil, err
  1001→		}
  1002→
  1003→		if rpoint.Bin != nil {
  1004→			if idx, ok := binMap[*rpoint.Bin]; ok {
  1005→				points[idx].Count = rpoint.Count
  1006→			}
  1007→		}
  1008→	}
  1009→
  1010→	if err := rows.Err(); err != nil {
  1011→		return nil, err
  1012→	}
  1013→
  1014→	result := model.MetricHistoPoints{Metric: metric, Unit: unit, Stat: &footprintStat, Data: points}
  1015→
  1016→	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
  1017→	return &result, nil
  1018→}
  1019→
  1020→// runningJobsMetricStatisticsHistogram generates metric histograms for running jobs using live data.
  1021→//
  1022→// Unlike completed jobs which use footprint data from the database, running jobs require
  1023→// fetching current metric averages from the metric backend (via metricdispatch).
  1024→//
  1025→// Parameters:
  1026→//   - metrics: List of metric names
  1027→//   - filters: Job filters (should filter to running jobs only)
  1028→//   - bins: Number of histogram bins
  1029→//
  1030→// Returns slice of MetricHistoPoints, one per metric.
  1031→//
  1032→// Limitations:
  1033→//   - Maximum 5000 jobs (returns nil if more jobs match)
  1034→//   - Requires metric backend availability
  1035→//   - Bins based on metric peak values from cluster configuration
  1036→//
  1037→// Algorithm:
  1038→//  1. Query first 5001 jobs to check count limit
  1039→//  2. Load metric averages for all jobs via metricdispatch
  1040→//  3. For each metric, create bins based on peak value
  1041→//  4. Iterate averages and count jobs per bin
  1042→func (r *JobRepository) runningJobsMetricStatisticsHistogram(
  1043→	ctx context.Context,
  1044→	metrics []string,
  1045→	filters []*model.JobFilter,
  1046→	bins *int,
  1047→) []*model.MetricHistoPoints {
  1048→	// Get Jobs
  1049→	jobs, err := r.QueryJobs(ctx, filters, &model.PageRequest{Page: 1, ItemsPerPage: 5000 + 1}, nil)
  1050→	if err != nil {
  1051→		cclog.Errorf("Error while querying jobs for footprint: %s", err)
  1052→		return nil
  1053→	}
  1054→	if len(jobs) > 5000 {
  1055→		cclog.Errorf("too many jobs matched (max: %d)", 5000)
  1056→		return nil
  1057→	}
  1058→
  1059→	// Get AVGs from metric repo
  1060→	avgs := make([][]schema.Float, len(metrics))
  1061→	for i := range avgs {
  1062→		avgs[i] = make([]schema.Float, 0, len(jobs))
  1063→	}
  1064→
  1065→	for _, job := range jobs {
  1066→		if job.MonitoringStatus == schema.MonitoringStatusDisabled || job.MonitoringStatus == schema.MonitoringStatusArchivingFailed {
  1067→			continue
  1068→		}
  1069→
  1070→		if err := metricdispatch.LoadAverages(job, metrics, avgs, ctx); err != nil {
  1071→			cclog.Errorf("Error while loading averages for histogram: %s", err)
  1072→			return nil
  1073→		}
  1074→	}
  1075→
  1076→	// Iterate metrics to fill endresult
  1077→	data := make([]*model.MetricHistoPoints, 0)
  1078→	for idx, metric := range metrics {
  1079→		// Get specific Peak or largest Peak
  1080→		var metricConfig *schema.MetricConfig
  1081→		var peak float64
  1082→		var unit string
  1083→
  1084→		for _, f := range filters {
  1085→			if f.Cluster != nil {
  1086→				metricConfig = archive.GetMetricConfig(*f.Cluster.Eq, metric)
  1087→				peak = metricConfig.Peak
  1088→				unit = metricConfig.Unit.Prefix + metricConfig.Unit.Base
  1089→			}
  1090→		}
  1091→
  1092→		if peak == 0.0 {
  1093→			for _, c := range archive.Clusters {
  1094→				for _, m := range c.MetricConfig {
  1095→					if m.Name == metric {
  1096→						if m.Peak > peak {
  1097→							peak = m.Peak
  1098→						}
  1099→						if unit == "" {
  1100→							unit = m.Unit.Prefix + m.Unit.Base
  1101→						}
  1102→					}
  1103→				}
  1104→			}
  1105→		}
  1106→
  1107→		// Make and fill bins
  1108→		peakBin := int(peak) / *bins
  1109→
  1110→		points := make([]*model.MetricHistoPoint, 0)
  1111→		for b := 0; b < *bins; b++ {
  1112→			count := 0
  1113→			bindex := b + 1
  1114→			bmin := peakBin * b
  1115→			bmax := peakBin * (b + 1)
  1116→
  1117→			// Iterate AVG values for indexed metric and count for bins
  1118→			for _, val := range avgs[idx] {
  1119→				if int(val) >= bmin && int(val) < bmax {
  1120→					count += 1
  1121→				}
  1122→			}
  1123→
  1124→			// Append Bin to Metric Result Array
  1125→			point := model.MetricHistoPoint{Bin: &bindex, Count: count, Min: &bmin, Max: &bmax}
  1126→			points = append(points, &point)
  1127→		}
  1128→
  1129→		// Append Metric Result Array to final results array
  1130→		result := model.MetricHistoPoints{Metric: metric, Unit: unit, Data: points}
  1131→		data = append(data, &result)
  1132→	}
  1133→
  1134→	return data
  1135→}
  1136→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"database/sql"
    10→	"embed"
    11→	"fmt"
    12→
    13→	cclog "github.com/ClusterCockpit/cc-lib/v2/ccLogger"
    14→	"github.com/golang-migrate/migrate/v4"
    15→	"github.com/golang-migrate/migrate/v4/database/sqlite3"
    16→	"github.com/golang-migrate/migrate/v4/source/iofs"
    17→)
    18→
    19→// Version is the current database schema version required by this version of cc-backend.
    20→// When the database schema changes, this version is incremented and a new migration file
    21→// is added to internal/repository/migrations/sqlite3/.
    22→//
    23→// Version history:
    24→//   - Version 12: Add covering index for stats queries (cluster, start_time, hpc_user, ...)
    25→//   - Version 11: Optimize job table indexes (reduce from ~78 to 48)
    26→//   - Version 10: Node table
    27→//
    28→// Migration files are embedded at build time from the migrations directory.
    29→const Version uint = 12
    30→
    31→//go:embed migrations/*
    32→var migrationFiles embed.FS
    33→
    34→// checkDBVersion verifies that the database schema version matches the expected version.
    35→// This is called automatically during Connect() to ensure schema compatibility.
    36→//
    37→// Returns an error if:
    38→//   - Database version is older than expected (needs migration)
    39→//   - Database version is newer than expected (needs app upgrade)
    40→//   - Database is in a dirty state (failed migration)
    41→//
    42→// A "dirty" database indicates a migration was started but not completed successfully.
    43→// This requires manual intervention to fix the database and force the version.
    44→func checkDBVersion(db *sql.DB) error {
    45→	driver, err := sqlite3.WithInstance(db, &sqlite3.Config{})
    46→	if err != nil {
    47→		return err
    48→	}
    49→	d, err := iofs.New(migrationFiles, "migrations/sqlite3")
    50→	if err != nil {
    51→		return err
    52→	}
    53→
    54→	m, err := migrate.NewWithInstance("iofs", d, "sqlite3", driver)
    55→	if err != nil {
    56→		return err
    57→	}
    58→
    59→	v, dirty, err := m.Version()
    60→	if err != nil {
    61→		if err == migrate.ErrNilVersion {
    62→			cclog.Warn("Legacy database without version or missing database file!")
    63→		} else {
    64→			return err
    65→		}
    66→	}
    67→
    68→	if v < Version {
    69→		return fmt.Errorf("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version)
    70→	} else if v > Version {
    71→		return fmt.Errorf("unsupported database version %d, need %d.\nPlease refer to documentation how to downgrade db with external migrate tool", v, Version)
    72→	}
    73→
    74→	if dirty {
    75→		return fmt.Errorf("last migration to version %d has failed, please fix the db manually and force version with -force-db flag", Version)
    76→	}
    77→
    78→	return nil
    79→}
    80→
    81→// getMigrateInstance creates a new migration instance for the given database file.
    82→// This is used internally by MigrateDB, RevertDB, and ForceDB.
    83→func getMigrateInstance(db string) (m *migrate.Migrate, err error) {
    84→	d, err := iofs.New(migrationFiles, "migrations/sqlite3")
    85→	if err != nil {
    86→		return nil, err
    87→	}
    88→
    89→	m, err = migrate.NewWithSourceInstance("iofs", d, fmt.Sprintf("sqlite3://%s?_foreign_keys=on", db))
    90→	if err != nil {
    91→		return nil, err
    92→	}
    93→
    94→	return m, nil
    95→}
    96→
    97→// MigrateDB applies all pending database migrations to bring the schema up to date.
    98→// This should be run with the -migrate-db flag before starting the application
    99→// after upgrading to a new version that requires schema changes.
   100→//
   101→// Process:
   102→//  1. Checks current database version
   103→//  2. Applies all migrations from current version to target Version
   104→//  3. Updates schema_migrations table to track applied migrations
   105→//
   106→// Important:
   107→//   - Always backup your database before running migrations
   108→//   - Migrations are irreversible without manual intervention
   109→//   - If a migration fails, the database is marked "dirty" and requires manual fix
   110→//
   111→// Usage:
   112→//
   113→//	cc-backend -migrate-db
   114→func MigrateDB(db string) error {
   115→	m, err := getMigrateInstance(db)
   116→	if err != nil {
   117→		return err
   118→	}
   119→
   120→	v, dirty, err := m.Version()
   121→	if err != nil {
   122→		if err == migrate.ErrNilVersion {
   123→			cclog.Info("Legacy database without version or missing database file!")
   124→		} else {
   125→			return err
   126→		}
   127→	}
   128→
   129→	if v < Version {
   130→		cclog.Infof("unsupported database version %d, need %d.\nPlease backup your database file and run cc-backend -migrate-db", v, Version)
   131→	}
   132→
   133→	if dirty {
   134→		return fmt.Errorf("last migration to version %d has failed, please fix the db manually and force version with -force-db flag", Version)
   135→	}
   136→
   137→	if err := m.Up(); err != nil {
   138→		if err == migrate.ErrNoChange {
   139→			cclog.Info("DB already up to date!")
   140→		} else {
   141→			return err
   142→		}
   143→	}
   144→
   145→	m.Close()
   146→	return nil
   147→}
   148→
   149→// RevertDB rolls back the database schema to the previous version (Version - 1).
   150→// This is primarily used for testing or emergency rollback scenarios.
   151→//
   152→// Warning:
   153→//   - This may cause data loss if newer schema added columns/tables
   154→//   - Always backup before reverting
   155→//   - Not all migrations are safely reversible
   156→//
   157→// Usage:
   158→//
   159→//	cc-backend -revert-db
   160→func RevertDB(db string) error {
   161→	m, err := getMigrateInstance(db)
   162→	if err != nil {
   163→		return err
   164→	}
   165→
   166→	if err := m.Migrate(Version - 1); err != nil {
   167→		if err == migrate.ErrNoChange {
   168→			cclog.Info("DB already up to date!")
   169→		} else {
   170→			return err
   171→		}
   172→	}
   173→
   174→	m.Close()
   175→	return nil
   176→}
   177→
   178→// ForceDB forces the database schema version to the current Version without running migrations.
   179→// This is only used to recover from failed migrations that left the database in a "dirty" state.
   180→//
   181→// When to use:
   182→//   - After manually fixing a failed migration
   183→//   - When you've manually applied schema changes and need to update the version marker
   184→//
   185→// Warning:
   186→//   - This does NOT apply any schema changes
   187→//   - Only use after manually verifying the schema is correct
   188→//   - Improper use can cause schema/version mismatch
   189→//
   190→// Usage:
   191→//
   192→//	cc-backend -force-db
   193→func ForceDB(db string) error {
   194→	m, err := getMigrateInstance(db)
   195→	if err != nil {
   196→		return err
   197→	}
   198→
   199→	if err := m.Force(int(Version)); err != nil {
   200→		return err
   201→	}
   202→
   203→	m.Close()
   204→	return nil
   205→}
   206→


     1→// Copyright (C) NHR@FAU, University Erlangen-Nuremberg.
     2→// All rights reserved. This file is part of cc-backend.
     3→// Use of this source code is governed by a MIT-style
     4→// license that can be found in the LICENSE file.
     5→
     6→package repository
     7→
     8→import (
     9→	"fmt"
    10→	"testing"
    11→
    12→	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
    13→)
    14→
    15→func TestBuildJobStatsQuery(t *testing.T) {
    16→	r := setup(t)
    17→	q := r.buildStatsQuery(nil, "USER", 300, nil)
    18→
    19→	sql, _, err := q.ToSql()
    20→	noErr(t, err)
    21→
    22→	fmt.Printf("SQL: %s\n", sql)
    23→}
    24→
    25→func TestJobStats(t *testing.T) {
    26→	r := setup(t)
    27→
    28→	var expectedCount int
    29→	err := r.DB.QueryRow(`SELECT COUNT(*) FROM job`).Scan(&expectedCount)
    30→	noErr(t, err)
    31→
    32→	stats, err := r.JobsStats(getContext(t), []*model.JobFilter{}, nil)
    33→	noErr(t, err)
    34→
    35→	if stats[0].TotalJobs != expectedCount {
    36→		t.Fatalf("Want %d, Got %d", expectedCount, stats[0].TotalJobs)
    37→	}
    38→}
    39→


     1→DROP INDEX IF EXISTS job_stats;
     2→DROP INDEX IF EXISTS job_by_user;
     3→DROP INDEX IF EXISTS job_by_starttime;
     4→DROP INDEX IF EXISTS job_by_job_id;
     5→DROP INDEX IF EXISTS job_list;
     6→DROP INDEX IF EXISTS job_list_user;
     7→DROP INDEX IF EXISTS job_list_users;
     8→DROP INDEX IF EXISTS job_list_users_start;
     9→
    10→ALTER TABLE job ADD COLUMN energy REAL NOT NULL DEFAULT 0.0;
    11→ALTER TABLE job ADD COLUMN energy_footprint TEXT DEFAULT NULL;
    12→
    13→ALTER TABLE job ADD COLUMN footprint TEXT DEFAULT NULL;
    14→ALTER TABLE tag ADD COLUMN tag_scope TEXT NOT NULL DEFAULT 'global';
    15→
    16→-- Do not use reserved keywords anymore
    17→ALTER TABLE "user" RENAME TO hpc_user;
    18→ALTER TABLE job RENAME COLUMN "user" TO hpc_user;
    19→ALTER TABLE job RENAME COLUMN "partition" TO cluster_partition;
    20→
    21→UPDATE job SET footprint = '{"flops_any_avg": 0.0}';
    22→UPDATE job SET footprint = json_replace(footprint, '$.flops_any_avg', job.flops_any_avg);
    23→UPDATE job SET footprint = json_insert(footprint, '$.mem_bw_avg', job.mem_bw_avg);
    24→UPDATE job SET footprint = json_insert(footprint, '$.mem_used_max', job.mem_used_max);
    25→UPDATE job SET footprint = json_insert(footprint, '$.cpu_load_avg', job.load_avg);
    26→UPDATE job SET footprint = json_insert(footprint, '$.net_bw_avg', job.net_bw_avg) WHERE job.net_bw_avg != 0;
    27→UPDATE job SET footprint = json_insert(footprint, '$.net_data_vol_total', job.net_data_vol_total) WHERE job.net_data_vol_total != 0;
    28→UPDATE job SET footprint = json_insert(footprint, '$.file_bw_avg', job.file_bw_avg) WHERE job.file_bw_avg != 0;
    29→UPDATE job SET footprint = json_insert(footprint, '$.file_data_vol_total', job.file_data_vol_total) WHERE job.file_data_vol_total != 0;
    30→
    31→ALTER TABLE job DROP flops_any_avg;
    32→ALTER TABLE job DROP mem_bw_avg;
    33→ALTER TABLE job DROP mem_used_max;
    34→ALTER TABLE job DROP load_avg;
    35→ALTER TABLE job DROP net_bw_avg;
    36→ALTER TABLE job DROP net_data_vol_total;
    37→ALTER TABLE job DROP file_bw_avg;
    38→ALTER TABLE job DROP file_data_vol_total;
    39→
    40→-- Indices for: Single filters, combined filters, sorting, sorting with filters
    41→-- Cluster Filter
    42→CREATE INDEX IF NOT EXISTS jobs_cluster ON job (cluster);
    43→CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user);
    44→CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project);
    45→CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster);
    46→-- Cluster Filter Sorting
    47→CREATE INDEX IF NOT EXISTS jobs_cluster_starttime ON job (cluster, start_time);
    48→CREATE INDEX IF NOT EXISTS jobs_cluster_duration ON job (cluster, duration);
    49→CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes);
    50→CREATE INDEX IF NOT EXISTS jobs_cluster_numhwthreads ON job (cluster, num_hwthreads);
    51→CREATE INDEX IF NOT EXISTS jobs_cluster_numacc ON job (cluster, num_acc);
    52→CREATE INDEX IF NOT EXISTS jobs_cluster_energy ON job (cluster, energy);
    53→
    54→-- Cluster+Partition Filter
    55→CREATE INDEX IF NOT EXISTS jobs_cluster_partition ON job (cluster, cluster_partition);
    56→-- Cluster+Partition Filter Sorting
    57→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, cluster_partition, start_time);
    58→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, cluster_partition, duration);
    59→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes);
    60→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads);
    61→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc);
    62→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy);
    63→
    64→-- Cluster+Partition+Jobstate Filter
    65→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state);
    66→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_user ON job (cluster, cluster_partition, job_state, hpc_user);
    67→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_project ON job (cluster, cluster_partition, job_state, project);
    68→-- Cluster+Partition+Jobstate Filter Sorting
    69→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_starttime ON job (cluster, cluster_partition, job_state, start_time);
    70→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_duration ON job (cluster, cluster_partition, job_state, duration);
    71→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numnodes ON job (cluster, cluster_partition, job_state, num_nodes);
    72→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numhwthreads ON job (cluster, cluster_partition, job_state, num_hwthreads);
    73→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numacc ON job (cluster, cluster_partition, job_state, num_acc);
    74→CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_energy ON job (cluster, cluster_partition, job_state, energy);
    75→
    76→-- Cluster+JobState Filter
    77→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate ON job (cluster, job_state);
    78→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user);
    79→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project);
    80→-- Cluster+JobState Filter Sorting
    81→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time);
    82→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration ON job (cluster, job_state, duration);
    83→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes);
    84→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numhwthreads ON job (cluster, job_state, num_hwthreads);
    85→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_state, num_acc);
    86→CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy);
    87→
    88→-- User Filter
    89→CREATE INDEX IF NOT EXISTS jobs_user ON job (hpc_user);
    90→-- User Filter Sorting
    91→CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time);
    92→CREATE INDEX IF NOT EXISTS jobs_user_duration ON job (hpc_user, duration);
    93→CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes);
    94→CREATE INDEX IF NOT EXISTS jobs_user_numhwthreads ON job (hpc_user, num_hwthreads);
    95→CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (hpc_user, num_acc);
    96→CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy);
    97→
    98→-- Project Filter
    99→CREATE INDEX IF NOT EXISTS jobs_project ON job (project);
   100→CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user);
   101→-- Project Filter Sorting
   102→CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time);
   103→CREATE INDEX IF NOT EXISTS jobs_project_duration ON job (project, duration);
   104→CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes);
   105→CREATE INDEX IF NOT EXISTS jobs_project_numhwthreads ON job (project, num_hwthreads);
   106→CREATE INDEX IF NOT EXISTS jobs_project_numacc ON job (project, num_acc);
   107→CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy);
   108→
   109→-- JobState Filter
   110→CREATE INDEX IF NOT EXISTS jobs_jobstate ON job (job_state);
   111→CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user);
   112→CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project);
   113→CREATE INDEX IF NOT EXISTS jobs_jobstate_cluster ON job (job_state, cluster);
   114→-- JobState Filter Sorting
   115→CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime ON job (job_state, start_time);
   116→CREATE INDEX IF NOT EXISTS jobs_jobstate_duration ON job (job_state, duration);
   117→CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes);
   118→CREATE INDEX IF NOT EXISTS jobs_jobstate_numhwthreads ON job (job_state, num_hwthreads);
   119→CREATE INDEX IF NOT EXISTS jobs_jobstate_numacc ON job (job_state, num_acc);
   120→CREATE INDEX IF NOT EXISTS jobs_jobstate_energy ON job (job_state, energy);
   121→
   122→-- ArrayJob Filter
   123→CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time);
   124→CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time);
   125→
   126→-- Sorting without active filters
   127→CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time);
   128→CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration);
   129→CREATE INDEX IF NOT EXISTS jobs_numnodes ON job (num_nodes);
   130→CREATE INDEX IF NOT EXISTS jobs_numhwthreads ON job (num_hwthreads);
   131→CREATE INDEX IF NOT EXISTS jobs_numacc ON job (num_acc);
   132→CREATE INDEX IF NOT EXISTS jobs_energy ON job (energy);
   133→
   134→-- Single filters with default starttime sorting
   135→CREATE INDEX IF NOT EXISTS jobs_duration_starttime ON job (duration, start_time);
   136→CREATE INDEX IF NOT EXISTS jobs_numnodes_starttime ON job (num_nodes, start_time);
   137→CREATE INDEX IF NOT EXISTS jobs_numhwthreads_starttime ON job (num_hwthreads, start_time);
   138→CREATE INDEX IF NOT EXISTS jobs_numacc_starttime ON job (num_acc, start_time);
   139→CREATE INDEX IF NOT EXISTS jobs_energy_starttime ON job (energy, start_time);
   140→
   141→-- Optimize DB index usage
   142→PRAGMA optimize;
   143→


     1→-- Migration 11: Remove overly specific table indexes formerly used in sorting
     2→-- When one or two indexed columns are used, sorting usually is fast
     3→-- Reduces from ~78 indexes to 48 for better write performance,
     4→-- reduced disk usage, and more reliable query planner decisions.
     5→-- Requires ANALYZE to be run after migration (done automatically on startup).
     6→
     7→-- ============================================================
     8→-- Drop SELECTED existing job indexes (from migrations 08/09)
     9→-- sqlite_autoindex_job_1 (UNIQUE constraint) is kept automatically
    10→-- ============================================================
    11→
    12→-- Cluster+Partition Filter Sorting
    13→DROP INDEX IF EXISTS jobs_cluster_partition_numnodes;
    14→DROP INDEX IF EXISTS jobs_cluster_partition_numhwthreads;
    15→DROP INDEX IF EXISTS jobs_cluster_partition_numacc;
    16→DROP INDEX IF EXISTS jobs_cluster_partition_energy;
    17→
    18→-- Cluster+JobState Filter Sorting
    19→DROP INDEX IF EXISTS jobs_cluster_jobstate_numnodes;
    20→DROP INDEX IF EXISTS jobs_cluster_jobstate_numhwthreads;
    21→DROP INDEX IF EXISTS jobs_cluster_jobstate_numacc;
    22→DROP INDEX IF EXISTS jobs_cluster_jobstate_energy;
    23→
    24→-- Cluster+Shared Filter Sorting
    25→DROP INDEX IF EXISTS jobs_cluster_shared_numnodes;
    26→DROP INDEX IF EXISTS jobs_cluster_shared_numhwthreads;
    27→DROP INDEX IF EXISTS jobs_cluster_shared_numacc;
    28→DROP INDEX IF EXISTS jobs_cluster_shared_energy;
    29→
    30→-- User Filter Sorting
    31→DROP INDEX IF EXISTS jobs_user_numnodes;
    32→DROP INDEX IF EXISTS jobs_user_numhwthreads;
    33→DROP INDEX IF EXISTS jobs_user_numacc;
    34→DROP INDEX IF EXISTS jobs_user_energy;
    35→
    36→-- Project Filter Sorting
    37→DROP INDEX IF EXISTS jobs_project_numnodes;
    38→DROP INDEX IF EXISTS jobs_project_numhwthreads;
    39→DROP INDEX IF EXISTS jobs_project_numacc;
    40→DROP INDEX IF EXISTS jobs_project_energy;
    41→
    42→-- JobState Filter Sorting
    43→DROP INDEX IF EXISTS jobs_jobstate_numnodes;
    44→DROP INDEX IF EXISTS jobs_jobstate_numhwthreads;
    45→DROP INDEX IF EXISTS jobs_jobstate_numacc;
    46→DROP INDEX IF EXISTS jobs_jobstate_energy;
    47→
    48→-- Shared Filter Sorting
    49→DROP INDEX IF EXISTS jobs_shared_numnodes;
    50→DROP INDEX IF EXISTS jobs_shared_numhwthreads;
    51→DROP INDEX IF EXISTS jobs_shared_numacc;
    52→DROP INDEX IF EXISTS jobs_shared_energy;
    53→
    54→-- ArrayJob Filter
    55→DROP INDEX IF EXISTS jobs_cluster_arrayjobid_starttime;
    56→
    57→-- Backup Indices For High Variety Columns
    58→DROP INDEX IF EXISTS jobs_duration;
    59→
    60→-- Optimize DB index usage
    61→PRAGMA optimize;
    62→


Found 1 file
internal/repository/jobQuery.go


36://   - filters: Optional job filters (cluster, state, user, time ranges, etc.)
179:// them to the query. Supports filtering by job properties (cluster, state, user),
191:		query = buildStringCondition("job.cluster", filter.Cluster, query)
194:		query = buildStringCondition("job.subcluster", filter.SubCluster, query)
197:		query = buildStringCondition("job.cluster_partition", filter.Partition, query)
227:	if filter.StartTime != nil {
228:		query = buildTimeCondition("job.start_time", filter.StartTime, query)
273:		// If a job completed within the configured timeframe, it will still show up after the start_time matches the condition!
274:		query = query.Where(sq.Lt{"job.start_time": (now - int64(*filter.MinRunningFor))})


147:	durationExpr := fmt.Sprintf(`(CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)`, now)
647:	value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as int) as value`, time.Now().Unix(), targetBinSize)


60:// groupBy2column maps GraphQL Aggregate enum values to their corresponding database column names.
63:var groupBy2column = map[model.Aggregate]string{
242:	col := groupBy2column[*groupBy]
440:	col := groupBy2column[*groupBy]
501:	col := groupBy2column[*groupBy]


Good. I have a complete picture now. Here is the detailed optimization plan.

---

## Detailed Optimization Plan

### 1. Index Column Reorder (Migration 12)

**The core problem**: The current index is `(cluster, start_time, hpc_user, duration, job_state, num_nodes, num_hwthreads, num_acc)`. For the query pattern `WHERE cluster = ? AND start_time >= ? GROUP BY hpc_user`, SQLite processes it as:

1. Use index prefix `cluster = ?` (equality) -- good
2. Scan `start_time >= ?` (range) -- good, but this is where ordered index traversal ends
3. For `GROUP BY hpc_user` -- SQLite cannot use index order because `hpc_user` comes AFTER the range column. It must collect all matching rows and sort them into a temp B-Tree.

**The fix**: Reorder to `(cluster, hpc_user, start_time, duration, job_state, num_nodes, num_hwthreads, num_acc)`.

With this order:
1. `cluster = ?` (equality on prefix) -- efficient
2. `GROUP BY hpc_user` -- the next column after the equality prefix, so SQLite can walk the index in order, producing groups naturally without a temp B-Tree
3. Within each `(cluster, hpc_user)` group, `start_time` is sorted, so `start_time >= ?` becomes a binary search within each group (skip to the first qualifying row and scan forward)
4. Remaining columns `(duration, job_state, num_nodes, num_hwthreads, num_acc)` make it a covering index -- no table lookups needed

**The query plan changes from**:
- Scan index for `(cluster, start_time >= ?)` -> collect all rows -> sort by `hpc_user` (temp B-Tree) -> aggregate

**To**:
- Walk index in `(cluster, hpc_user)` order -> within each group, skip-scan to `start_time >= ?` -> aggregate directly from index

For 300k+ jobs over 30 days, eliminating the temp B-Tree sort is the critical win.

**New migration SQL** for `12_stats-covering-index.up.sql`:

```sql
-- Migration 12: Add covering index for grouped stats queries
-- Column order: cluster (equality), hpc_user (GROUP BY), start_time (range filter)
-- followed by aggregation columns to make it a covering index.
-- This order lets SQLite produce GROUP BY output directly from index order,
-- avoiding an expensive temp B-Tree sort.

DROP INDEX IF EXISTS jobs_cluster_starttime_user_stats;

CREATE INDEX IF NOT EXISTS jobs_cluster_user_starttime_stats
  ON job (cluster, hpc_user, start_time, duration, job_state, num_nodes, num_hwthreads, num_acc);

PRAGMA optimize;
```

Note: The `DROP INDEX` is needed because this migration may run on databases that already applied the old version 12. The index name changes to reflect the new column order.

**Down migration** `12_stats-covering-index.down.sql`:

```sql
DROP INDEX IF EXISTS jobs_cluster_user_starttime_stats;
```

### 2. Code Changes in stats.go

**Change A: `COUNT(job.id)` to `COUNT(*)`**

Line 156: Change `"COUNT(job.id) as totalJobs"` to `"COUNT(*) as totalJobs"`

Line 108 (in `buildCountQuery`): Change `"COUNT(job.id)"` to `"COUNT(*)"` in both branches.

Lines 751, 817: In `jobsStatisticsHistogram` and `jobsDurationStatisticsHistogram`, change `"COUNT(job.id) AS count"` to `"COUNT(*) AS count"`.

**Reasoning**: `COUNT(*)` is semantically clearer for "count all rows in each group." With `COUNT(job.id)`, SQLite must technically check that `job.id` is not NULL for each row (even though the rowid can never be NULL). While SQLite likely optimizes this away, `COUNT(*)` is the idiomatic form and removes any ambiguity. In a covering index scan where the rowid is implicitly present but not explicitly listed, `COUNT(*)` is the safer choice.

**Change B: Fix double-quoted string literals**

Line 147: Change `"running"` (double quotes) to `''running''` (single quotes inside the backtick raw string):

```go
durationExpr := fmt.Sprintf(`(CASE WHEN job.job_state = 'running' THEN %d - job.start_time ELSE job.duration END)`, now)
```

Line 647 (in `AddHistograms`): Same fix:

```go
value := fmt.Sprintf(`CAST(ROUND(((CASE WHEN job.job_state = 'running' THEN %d - job.start_time ELSE job.duration END) / %d) + 1) as int) as value`, time.Now().Unix(), targetBinSize)
```

**Reasoning**: In SQL, double quotes denote identifiers (table/column names), not string literals. SQLite has a compatibility quirk that accepts double-quoted strings when no matching identifier exists, but this is non-standard. Using proper single quotes avoids any risk of the query planner misinterpreting the token as a column reference, which could affect index usage decisions.

### 3. Whether to Drop `jobs_cluster_starttime`

**Recommendation: Do NOT drop it in this migration.**

The `jobs_cluster_starttime ON (cluster, start_time)` index from migration 08 serves a different purpose -- it supports the very common job list query pattern `WHERE cluster = ? ORDER BY start_time DESC LIMIT N` (the default job list view). The new covering index `(cluster, hpc_user, start_time, ...)` has `hpc_user` between `cluster` and `start_time`, so it cannot efficiently serve `ORDER BY start_time` without a user filter.

The `jobs_cluster_user ON (cluster, hpc_user)` index from migration 08 IS made redundant by the new covering index (which has `(cluster, hpc_user, ...)` as its prefix). However, the overhead of one small 2-column index is negligible, and dropping it would require careful analysis of all query paths. This can be a follow-up cleanup if desired.

**Summary**: Both existing indexes serve query patterns that the new covering index does not optimally cover. Leave them.

### 4. Other GROUP BY Dimensions

The `groupBy2column` map shows four possible grouping columns: `hpc_user`, `project`, `cluster`, `subcluster`. The dominant use case driving this fix is the users list page (`GROUP BY hpc_user` with `cluster` + `start_time` filters). The other dimensions are less common and typically faster (fewer distinct values for cluster/subcluster, or project queries use different filter patterns). Creating additional covering indexes for every GROUP BY variant would be counterproductive -- more indexes slow writes and consume disk. The single reordered index addresses the primary bottleneck.

### 5. Testing and Verification Approach

**Unit tests** (no changes needed to test file, existing tests cover correctness):

```bash
go test -v ./internal/repository -run TestJobStats
go test -v ./internal/repository -run TestBuildJobStatsQuery
```

The `TestBuildJobStatsQuery` test prints the generated SQL -- verify it now uses `COUNT(*)` and single-quoted `'running'`.

**Query plan verification on production database** (before/after):

```sql
-- Before: check current plan
EXPLAIN QUERY PLAN
SELECT job.hpc_user, COUNT(*) as totalJobs, ...
FROM job
WHERE job.cluster = 'woody' AND job.start_time >= 1709251200
GROUP BY job.hpc_user;
```

Before the fix, you will see `USE TEMP B-TREE FOR GROUP BY` in the output. After applying the new index, this line should disappear, replaced by direct index scan.

**Performance measurement**:

```sql
-- Time the actual query (SQLite CLI)
.timer on
SELECT job.hpc_user, COUNT(*) as totalJobs, ...
FROM job
WHERE job.cluster = 'woody' AND job.start_time >= 1709251200
GROUP BY job.hpc_user;
```

Expected improvement: from "does not return" (minutes+) to sub-second, because the temp B-Tree sort over hundreds of thousands of rows is eliminated.

**Migration testing**:

1. Test on a fresh database: `cc-backend -migrate-db` should apply all 12 migrations cleanly.
2. Test on existing v11 database: migration 12 should create the index with new column order.
3. Test on existing v12 database (old index): the `DROP INDEX IF EXISTS` + `CREATE INDEX` handles this case -- the old index is dropped and new one created.

### 6. Implementation Sequence

1. Modify `12_stats-covering-index.up.sql` -- reorder columns, add DROP for old index name, rename index
2. Modify `12_stats-covering-index.down.sql` -- update to drop the new index name
3. Modify `stats.go` -- `COUNT(job.id)` to `COUNT(*)`, fix double-quoted string literals
4. Run tests: `go test ./internal/repository/...`
5. Verify generated SQL with `TestBuildJobStatsQuery`
6. Test migration on a copy of production database
7. Verify query plan with `EXPLAIN QUERY PLAN`

### Critical Files for Implementation

- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/migrations/sqlite3/12_stats-covering-index.up.sql` - Reorder index columns from `(cluster, start_time, hpc_user, ...)` to `(cluster, hpc_user, start_time, ...)`
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/migrations/sqlite3/12_stats-covering-index.down.sql` - Update to drop the renamed index
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/stats.go` - Fix COUNT(job.id) to COUNT(*), fix double-quoted 'running' string literals (lines 108, 147, 156, 647, 751, 817)
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/stats_test.go` - Existing tests to run for validation (no changes needed)
- `/Users/jan/prg/ClusterCockpit/cc-backend/internal/repository/migration.go` - No changes needed (Version stays at 12), but important reference for migration framework

agentId: a9b266f25baa3e441 (for resuming to continue this agent's work if needed)
<usage>total_tokens: 40684
tool_uses: 13
duration_ms: 86652</usage>

Prompt:
I need to optimize a slow SQLite stats query in a Go backend (ClusterCockpit cc-backend). Here's the situation:

## Problem
The endpoint `/monitoring/users/?cluster=woody&startTime=last30d` calls `buildStatsQuery()` which generates a SQL query like:

```sql
SELECT job.hpc_user,
  COUNT(job.id) as totalJobs,
  0 AS totalUsers,
  CAST(ROUND(SUM(CASE WHEN job.job_state = "running" THEN <now> - job.start_time ELSE job.duration END) / 3600) as int) as totalWalltime,
  ...similar aggregations for totalCoreHours, totalAccHours...
  COUNT(CASE WHEN job.duration < 300 THEN 1 END) as shortJobs
FROM job
WHERE job.cluster = ? AND job.start_time >= ?
GROUP BY job.hpc_user
```

The query "does not return" - essentially hangs on a production database.

## Current Index (Migration 12)
```sql
CREATE INDEX jobs_cluster_starttime_user_stats
  ON job (cluster, start_time, hpc_user, duration, job_state, num_nodes, num_hwthreads, num_acc);
```

## Other relevant indexes
- `jobs_cluster_starttime ON (cluster, start_time)` - from migration 08
- `jobs_cluster_user ON (cluster, hpc_user)` - from migration 08

## Key Observations

1. **Index column order problem**: The covering index has `(cluster, start_time, hpc_user, ...)`. The WHERE clause uses `cluster = ?` (equality) and `start_time >= ?` (range). The GROUP BY is on `hpc_user`. Since `start_time` is a range condition BEFORE `hpc_user`, the index can't provide ordered GROUP BY output. SQLite must sort all matching rows into a temp B-Tree before aggregating. For hundreds of thousands of jobs in 30 days, this sort is extremely expensive.

2. **Better index order**: `(cluster, hpc_user, start_time, ...)` would allow:
   - Equality on cluster (prefix)
   - Index-ordered GROUP BY on hpc_user (next column)
   - Within each (cluster, hpc_user) group, start_time is sorted so the >= filter uses efficient skip-scan
   - All aggregation columns are in the index (covering)

3. **COUNT(job.id)**: Uses the rowid which is implicitly in the index, but `COUNT(*)` is clearer and avoids any potential table lookups.

4. **Double quotes in CASE**: `job.job_state = "running"` uses double quotes (SQL identifiers). Should be single quotes for string literals, though SQLite handles this gracefully.

## Files to modify
- `internal/repository/migrations/sqlite3/12_stats-covering-index.up.sql` - reorder index columns
- `internal/repository/stats.go` - change COUNT(job.id) to COUNT(*), fix double quotes

Please provide a detailed plan for these optimizations, including:
1. The exact index change (with reasoning)
2. Code changes in stats.go
3. Whether we should drop the redundant `jobs_cluster_starttime` index (since we have a covering index that starts with cluster)
4. Testing/verification approach


Let me verify the exact lines in stats.go that need `COUNT(job.id)` fixes, and check the down migration.


     1→DROP INDEX IF EXISTS jobs_cluster_starttime_user_stats;
     2→


93://   - Single count: COUNT(job.id) when col is empty
94://   - Grouped counts: col, COUNT(job.id) when col is specified
108:		query = sq.Select(col, "COUNT(job.id)").From("job").GroupBy(col)
110:		query = sq.Select("COUNT(job.id)").From("job")
156:	columns = append(columns, "COUNT(job.id) as totalJobs")
751:		sq.Select(value, "COUNT(job.id) AS count").From("job"))
817:		sq.Select(value, "COUNT(job.id) AS count").From("job"))


Now I have all the details. Let me write the final plan.


Write

# Fix: buildStatsQuery hangs on `/monitoring/users/?cluster=woody&startTime=last30d`

## Context

The users list page (`/monitoring/users/?cluster=woody&startTime=last30d`) executes a `GROUP BY job.hpc_user` stats query with `WHERE job.cluster = ? AND job.start_time >= ?`. On production databases with hundreds of thousands of jobs in the last 30 days, this query never returns.

**Root cause**: The covering index from migration 12 has column order `(cluster, start_time, hpc_user, ...)`. Since `start_time` is a **range** condition before `hpc_user`, SQLite cannot use index ordering for `GROUP BY hpc_user`. It must collect all matching rows into a **temp B-Tree** and sort them — extremely expensive for large result sets.

## Changes

### 1. Reorder covering index columns (critical fix)

**File**: `internal/repository/migrations/sqlite3/12_stats-covering-index.up.sql`

Change from: `(cluster, start_time, hpc_user, duration, job_state, num_nodes, num_hwthreads, num_acc)`
Change to: `(cluster, hpc_user, start_time, duration, job_state, num_nodes, num_hwthreads, num_acc)`

This lets SQLite:
- Match `cluster = ?` (equality prefix)
- Walk `hpc_user` in index order (GROUP BY without temp B-Tree)
- Within each `(cluster, hpc_user)` group, binary-search `start_time >= ?`
- Read all aggregation columns from the index (covering)

Add `DROP INDEX IF EXISTS jobs_cluster_starttime_user_stats` to handle databases that already applied the old migration 12.

**File**: `internal/repository/migrations/sqlite3/12_stats-covering-index.down.sql`

Update to drop the renamed index `jobs_cluster_user_starttime_stats`.

### 2. Use `COUNT(*)` instead of `COUNT(job.id)`

**File**: `internal/repository/stats.go`

Lines to change: 108, 110, 156, 751, 817

`COUNT(*)` is idiomatic for counting rows and avoids any ambiguity about whether the rowid is available in a covering index scan.

### 3. Fix double-quoted string literals in CASE expressions

**File**: `internal/repository/stats.go`

Lines 147, 647: Change `job.job_state = "running"` to `job.job_state = 'running'`

Double quotes are SQL identifiers, not string literals. SQLite accepts them as a compatibility quirk but proper single quotes are safer for the query planner.

## Not changing

- **`jobs_cluster_starttime` index**: Still needed for job list queries (`ORDER BY start_time`).
- **`jobs_cluster_user` index**: Technically redundant with new covering index prefix, but harmless. Can clean up later.
- **Migration version**: Stays at 12 (same migration number, updated content).

## Verification

```bash
# Run existing tests
go test -v ./internal/repository -run TestJobStats
go test -v ./internal/repository -run TestBuildJobStatsQuery

# On production DB copy, verify query plan no longer shows temp B-Tree:
# EXPLAIN QUERY PLAN SELECT job.hpc_user, COUNT(*) ... FROM job
#   WHERE job.cluster = 'woody' AND job.start_time >= <30d_ago> GROUP BY job.hpc_user;
# Before: "USE TEMP B-TREE FOR GROUP BY"
# After: direct index scan, no temp B-Tree
```


File created successfully at: /Users/jan/.claude/plans/calm-sniffing-frog.md

# Fix: buildStatsQuery hangs on `/monitoring/users/?cluster=woody&startTime=last30d`

## Context

The users list page (`/monitoring/users/?cluster=woody&startTime=last30d`) executes a `GROUP BY job.hpc_user` stats query with `WHERE job.cluster = ? AND job.start_time >= ?`. On production databases with hundreds of thousands of jobs in the last 30 days, this query never returns.

**Root cause**: The covering index from migration 12 has column order `(cluster, start_time, hpc_user, ...)`. Since `start_time` is a **range** condition before `hpc_user`, SQLite cannot use index ordering for `GROUP BY hpc_user`. It must collect all matching rows into a **temp B-Tree** and sort them — extremely expensive for large result sets.

## Changes

### 1. Reorder covering index columns (critical fix)

**File**: `internal/repository/migrations/sqlite3/12_stats-covering-index.up.sql`

Change from: `(cluster, start_time, hpc_user, duration, job_state, num_nodes, num_hwthreads, num_acc)`
Change to: `(cluster, hpc_user, start_time, duration, job_state, num_nodes, num_hwthreads, num_acc)`

This lets SQLite:
- Match `cluster = ?` (equality prefix)
- Walk `hpc_user` in index order (GROUP BY without temp B-Tree)
- Within each `(cluster, hpc_user)` group, binary-search `start_time >= ?`
- Read all aggregation columns from the index (covering)

Add `DROP INDEX IF EXISTS jobs_cluster_starttime_user_stats` to handle databases that already applied the old migration 12.

**File**: `internal/repository/migrations/sqlite3/12_stats-covering-index.down.sql`

Update to drop the renamed index `jobs_cluster_user_starttime_stats`.

### 2. Use `COUNT(*)` instead of `COUNT(job.id)`

**File**: `internal/repository/stats.go`

Lines to change: 108, 110, 156, 751, 817

`COUNT(*)` is idiomatic for counting rows and avoids any ambiguity about whether the rowid is available in a covering index scan.

### 3. Fix double-quoted string literals in CASE expressions

**File**: `internal/repository/stats.go`

Lines 147, 647: Change `job.job_state = "running"` to `job.job_state = 'running'`

Double quotes are SQL identifiers, not string literals. SQLite accepts them as a compatibility quirk but proper single quotes are safer for the query planner.

## Not changing

- **`jobs_cluster_starttime` index**: Still needed for job list queries (`ORDER BY start_time`).
- **`jobs_cluster_user` index**: Technically redundant with new covering index prefix, but harmless. Can clean up later.
- **Migration version**: Stays at 12 (same migration number, updated content).

## Verification

```bash
# Run existing tests
go test -v ./internal/repository -run TestJobStats
go test -v ./internal/repository -run TestBuildJobStatsQuery

# On production DB copy, verify query plan no longer shows temp B-Tree:
# EXPLAIN QUERY PLAN SELECT job.hpc_user, COUNT(*) ... FROM job
#   WHERE job.cluster = 'woody' AND job.start_time >= <30d_ago> GROUP BY job.hpc_user;
# Before: "USE TEMP B-TREE FOR GROUP BY"
# After: direct index scan, no temp B-Tree
```


ToolSearch

Question: select:ExitPlanMode


Question: select:ExitPlanMode


The plan is written. The key fix is reordering the covering index columns from `(cluster, start_time, hpc_user, ...)` to `(cluster, hpc_user, start_time, ...)` so SQLite can produce GROUP BY output directly from the index order, eliminating the expensive temp B-Tree sort that causes the query to hang.


ExitPlanMode

# Fix: buildStatsQuery hangs on `/monitoring/users/?cluster=woody&startTime=last30d`

## Context

The users list page (`/monitoring/users/?cluster=woody&startTime=last30d`) executes a `GROUP BY job.hpc_user` stats query with `WHERE job.cluster = ? AND job.start_time >= ?`. On production databases with hundreds of thousands of jobs in the last 30 days, this query never returns.

**Root cause**: The covering index from migration 12 has column order `(cluster, start_time, hpc_user, ...)`. Since `start_time` is a **range** condition before `hpc_user`, SQLite cannot use index ordering for `GROUP BY hpc_user`. It must collect all matching rows into a **temp B-Tree** and sort them — extremely expensive for large result sets.

## Changes

### 1. Reorder covering index columns (critical fix)

**File**: `internal/repository/migrations/sqlite3/12_stats-covering-index.up.sql`

Change from: `(cluster, start_time, hpc_user, duration, job_state, num_nodes, num_hwthreads, num_acc)`
Change to: `(cluster, hpc_user, start_time, duration, job_state, num_nodes, num_hwthreads, num_acc)`

This lets SQLite:
- Match `cluster = ?` (equality prefix)
- Walk `hpc_user` in index order (GROUP BY without temp B-Tree)
- Within each `(cluster, hpc_user)` group, binary-search `start_time >= ?`
- Read all aggregation columns from the index (covering)

Add `DROP INDEX IF EXISTS jobs_cluster_starttime_user_stats` to handle databases that already applied the old migration 12.

**File**: `internal/repository/migrations/sqlite3/12_stats-covering-index.down.sql`

Update to drop the renamed index `jobs_cluster_user_starttime_stats`.

### 2. Use `COUNT(*)` instead of `COUNT(job.id)`

**File**: `internal/repository/stats.go`

Lines to change: 108, 110, 156, 751, 817

`COUNT(*)` is idiomatic for counting rows and avoids any ambiguity about whether the rowid is available in a covering index scan.

### 3. Fix double-quoted string literals in CASE expressions

**File**: `internal/repository/stats.go`

Lines 147, 647: Change `job.job_state = "running"` to `job.job_state = 'running'`

Double quotes are SQL identifiers, not string literals. SQLite accepts them as a compatibility quirk but proper single quotes are safer for the query planner.

## Not changing

- **`jobs_cluster_starttime` index**: Still needed for job list queries (`ORDER BY start_time`).
- **`jobs_cluster_user` index**: Technically redundant with new covering index prefix, but harmless. Can clean up later.
- **Migration version**: Stays at 12 (same migration number, updated content).

## Verification

```bash
# Run existing tests
go test -v ./internal/repository -run TestJobStats
go test -v ./internal/repository -run TestBuildJobStatsQuery

# On production DB copy, verify query plan no longer shows temp B-Tree:
# EXPLAIN QUERY PLAN SELECT job.hpc_user, COUNT(*) ... FROM job
#   WHERE job.cluster = 'woody' AND job.start_time >= <30d_ago> GROUP BY job.hpc_user;
# Before: "USE TEMP B-TREE FOR GROUP BY"
# After: direct index scan, no temp B-Tree
```


Error: The user doesn't want to proceed with this tool use. The tool use was rejected (eg. if it was a file edit, the new_string was NOT written to the file). STOP what you are doing and wait for the user to tell you how to proceed.


I'll wait for your feedback on the plan.