Merge branch 'hotfix' of github.com:ClusterCockpit/cc-backend into hotfix

internal/repository/dbConnection.go

@@ -48,6 +48,14 @@ func setupSqlite(db *sql.DB) error {
 		}
 	}
 
+	// Update query planner statistics so SQLite picks optimal indexes.
+	// Without this, SQLite guesses row distributions and often chooses wrong
+	// indexes for queries with IN clauses + ORDER BY, causing full table sorts
+	// in temp B-trees instead of using covering indexes.
+	if _, err := db.Exec("ANALYZE"); err != nil {
+		cclog.Warnf("Failed to run ANALYZE: %v", err)
+	}
+
 	return nil
 }
 

internal/repository/jobQuery.go

@@ -63,7 +63,7 @@ func (r *JobRepository) QueryJobs(
 			}
 		} else {
 			// Order by footprint JSON field values
-			query = query.Where("JSON_VALID(meta_data)")
+			query = query.Where("JSON_VALID(footprint)")
 			switch order.Order {
 			case model.SortDirectionEnumAsc:
 				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") ASC", field))
@@ -335,14 +335,14 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui
 }
 
 // buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column, using BETWEEN only if required.
-func buildFloatJSONCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
+func buildFloatJSONCondition(jsonField string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
 	query = query.Where("JSON_VALID(footprint)")
 	if cond.From != 1.0 && cond.To != 0.0 {
-		return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") BETWEEN ? AND ?", cond.From, cond.To)
+		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") BETWEEN ? AND ?", cond.From, cond.To)
 	} else if cond.From != 1.0 && cond.To == 0.0 {
-		return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") >= ?", cond.From)
+		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") >= ?", cond.From)
 	} else if cond.From == 1.0 && cond.To != 0.0 {
-		return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") <= ?", cond.To)
+		return query.Where("JSON_EXTRACT(footprint, \"$."+jsonField+"\") <= ?", cond.To)
 	} else {
 		return query
 	}

internal/repository/node.go

@@ -366,19 +366,23 @@ func (r *NodeRepository) QueryNodes(
 		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
 		return nil, err
 	}
+	defer rows.Close()
 
 	nodes := make([]*schema.Node, 0)
 	for rows.Next() {
 		node := schema.Node{}
 		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
 			&node.NodeState, &node.HealthState); err != nil {
-			rows.Close()
 			cclog.Warn("Error while scanning rows (QueryNodes)")
 			return nil, err
 		}
 		nodes = append(nodes, &node)
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	return nodes, nil
 }
 
@@ -415,6 +419,7 @@ func (r *NodeRepository) QueryNodesWithMeta(
 		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
 		return nil, err
 	}
+	defer rows.Close()
 
 	nodes := make([]*schema.Node, 0)
 	for rows.Next() {
@@ -424,7 +429,6 @@ func (r *NodeRepository) QueryNodesWithMeta(
 
 		if err := rows.Scan(&node.Hostname, &node.Cluster, &node.SubCluster,
 			&node.NodeState, &node.HealthState, &RawMetaData, &RawMetricHealth); err != nil {
-			rows.Close()
 			cclog.Warn("Error while scanning rows (QueryNodes)")
 			return nil, err
 		}
@@ -454,6 +458,10 @@ func (r *NodeRepository) QueryNodesWithMeta(
 		nodes = append(nodes, &node)
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	return nodes, nil
 }
 
@@ -545,10 +553,11 @@ func (r *NodeRepository) MapNodes(cluster string) (map[string]string, error) {
 
 func (r *NodeRepository) CountStates(ctx context.Context, filters []*model.NodeFilter, column string) ([]*model.NodeStates, error) {
 	query, qerr := AccessCheck(ctx,
-		sq.Select(column).
+		sq.Select(column, "COUNT(*) as count").
 			From("node").
 			Join("node_state ON node_state.node_id = node.id").
-			Where(latestStateCondition()))
+			Where(latestStateCondition()).
+			GroupBy(column))
 	if qerr != nil {
 		return nil, qerr
 	}
@@ -561,23 +570,21 @@ func (r *NodeRepository) CountStates(ctx context.Context, filters []*model.NodeF
 		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
 		return nil, err
 	}
+	defer rows.Close()
 
-	stateMap := map[string]int{}
+	nodes := make([]*model.NodeStates, 0)
 	for rows.Next() {
 		var state string
-		if err := rows.Scan(&state); err != nil {
-			rows.Close()
+		var count int
+		if err := rows.Scan(&state, &count); err != nil {
 			cclog.Warn("Error while scanning rows (CountStates)")
 			return nil, err
 		}
-
-		stateMap[state] += 1
+		nodes = append(nodes, &model.NodeStates{State: state, Count: count})
 	}
 
-	nodes := make([]*model.NodeStates, 0)
-	for state, counts := range stateMap {
-		node := model.NodeStates{State: state, Count: counts}
-		nodes = append(nodes, &node)
+	if err := rows.Err(); err != nil {
+		return nil, err
 	}
 
 	return nodes, nil
@@ -623,6 +630,7 @@ func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model.
 		cclog.Errorf("Error while running query '%s' %v: %v", queryString, queryVars, err)
 		return nil, err
 	}
+	defer rows.Close()
 
 	rawData := make(map[string][][]int)
 	for rows.Next() {
@@ -630,7 +638,6 @@ func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model.
 		var timestamp, count int
 
 		if err := rows.Scan(&state, &timestamp, &count); err != nil {
-			rows.Close()
 			cclog.Warnf("Error while scanning rows (CountStatesTimed) at time '%d'", timestamp)
 			return nil, err
 		}
@@ -643,6 +650,10 @@ func (r *NodeRepository) CountStatesTimed(ctx context.Context, filters []*model.
 		rawData[state][1] = append(rawData[state][1], count)
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	timedStates := make([]*model.NodeStatesTimed, 0)
 	for state, data := range rawData {
 		entry := model.NodeStatesTimed{State: state, Times: data[0], Counts: data[1]}

internal/repository/stats.go

@@ -235,6 +235,7 @@ func (r *JobRepository) JobsStatsGrouped(
 		cclog.Warn("Error while querying DB for job statistics")
 		return nil, err
 	}
+	defer rows.Close()
 
 	stats := make([]*model.JobsStatistics, 0, 100)
 
@@ -320,6 +321,10 @@ func (r *JobRepository) JobsStatsGrouped(
 		}
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	cclog.Debugf("Timer JobsStatsGrouped %s", time.Since(start))
 	return stats, nil
 }
@@ -440,6 +445,7 @@ func (r *JobRepository) JobCountGrouped(
 		cclog.Warn("Error while querying DB for job statistics")
 		return nil, err
 	}
+	defer rows.Close()
 
 	stats := make([]*model.JobsStatistics, 0, 100)
 
@@ -459,6 +465,10 @@ func (r *JobRepository) JobCountGrouped(
 		}
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	cclog.Debugf("Timer JobCountGrouped %s", time.Since(start))
 	return stats, nil
 }
@@ -496,6 +506,7 @@ func (r *JobRepository) AddJobCountGrouped(
 		cclog.Warn("Error while querying DB for job statistics")
 		return nil, err
 	}
+	defer rows.Close()
 
 	counts := make(map[string]int)
 
@@ -511,6 +522,10 @@ func (r *JobRepository) AddJobCountGrouped(
 		}
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	switch kind {
 	case "running":
 		for _, s := range stats {
@@ -550,23 +565,13 @@ func (r *JobRepository) AddJobCount(
 	if err != nil {
 		return nil, err
 	}
-	rows, err := query.RunWith(r.DB).Query()
-	if err != nil {
-		cclog.Warn("Error while querying DB for job statistics")
+	var cnt sql.NullInt64
+	if err := query.RunWith(r.DB).QueryRow().Scan(&cnt); err != nil {
+		cclog.Warn("Error while querying DB for job count")
 		return nil, err
 	}
 
-	var count int
-
-	for rows.Next() {
-		var cnt sql.NullInt64
-		if err := rows.Scan(&cnt); err != nil {
-			cclog.Warn("Error while scanning rows")
-			return nil, err
-		}
-
-		count = int(cnt.Int64)
-	}
+	count := int(cnt.Int64)
 
 	switch kind {
 	case "running":
@@ -755,6 +760,7 @@ func (r *JobRepository) jobsStatisticsHistogram(
 		cclog.Error("Error while running query")
 		return nil, err
 	}
+	defer rows.Close()
 
 	points := make([]*model.HistoPoint, 0)
 	// is it possible to introduce zero values here? requires info about bincount
@@ -767,6 +773,11 @@ func (r *JobRepository) jobsStatisticsHistogram(
 
 		points = append(points, &point)
 	}
+
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
 	return points, nil
 }
@@ -823,6 +834,7 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
 		cclog.Error("Error while running query")
 		return nil, err
 	}
+	defer rows.Close()
 
 	// Match query results to pre-initialized bins.
 	// point.Value from query is the bin number; multiply by binSizeSeconds to match bin.Value.
@@ -841,6 +853,10 @@ func (r *JobRepository) jobsDurationStatisticsHistogram(
 		}
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))
 	return points, nil
 }
@@ -948,6 +964,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 		cclog.Errorf("Error while running mainQuery: %s", err)
 		return nil, err
 	}
+	defer rows.Close()
 
 	// Pre-initialize bins with calculated min/max ranges.
 	// Example: peak=1000, bins=10 -> bin 1=[0,100), bin 2=[100,200), ..., bin 10=[900,1000]
@@ -976,6 +993,10 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 		}
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	result := model.MetricHistoPoints{Metric: metric, Unit: unit, Stat: &footprintStat, Data: points}
 
 	cclog.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start))

internal/repository/tags.go

@@ -283,6 +283,7 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
 	if err != nil {
 		return nil, nil, err
 	}
+	defer xrows.Close()
 
 	for xrows.Next() {
 		var t schema.Tag
@@ -300,6 +301,10 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
 		}
 	}
 
+	if err := xrows.Err(); err != nil {
+		return nil, nil, err
+	}
+
 	// Query and Count Jobs with attached Tags
 	q := sq.Select("t.tag_type, t.tag_name, t.id, count(jt.tag_id)").
 		From("tag t").
@@ -334,6 +339,7 @@ func (r *JobRepository) CountTags(user *schema.User) (tags []schema.Tag, counts
 	if err != nil {
 		return nil, nil, err
 	}
+	defer rows.Close()
 
 	counts = make(map[string]int)
 	for rows.Next() {
@@ -511,6 +517,7 @@ func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, e
 		cclog.Errorf("Error get tags with %s: %v", s, err)
 		return nil, err
 	}
+	defer rows.Close()
 
 	tags := make([]*schema.Tag, 0)
 	for rows.Next() {
@@ -529,6 +536,10 @@ func (r *JobRepository) GetTags(user *schema.User, job *int64) ([]*schema.Tag, e
 		}
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	return tags, nil
 }
 
@@ -544,6 +555,7 @@ func (r *JobRepository) GetTagsDirect(job *int64) ([]*schema.Tag, error) {
 		cclog.Errorf("Error get tags with %s: %v", s, err)
 		return nil, err
 	}
+	defer rows.Close()
 
 	tags := make([]*schema.Tag, 0)
 	for rows.Next() {
@@ -555,6 +567,10 @@ func (r *JobRepository) GetTagsDirect(job *int64) ([]*schema.Tag, error) {
 		tags = append(tags, tag)
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	return tags, nil
 }
 
@@ -582,6 +598,7 @@ func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
 		cclog.Errorf("Error get tags with %s: %v", s, err)
 		return nil, err
 	}
+	defer rows.Close()
 
 	tags := make([]*schema.Tag, 0)
 	for rows.Next() {
@@ -593,6 +610,10 @@ func (r *JobRepository) getArchiveTags(job *int64) ([]*schema.Tag, error) {
 		tags = append(tags, tag)
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
 	return tags, nil
 }
 

internal/repository/user.go

@@ -102,6 +102,7 @@ func (r *UserRepository) GetLdapUsernames() ([]string, error) {
 		cclog.Warn("Error while querying usernames")
 		return nil, err
 	}
+	defer rows.Close()
 
 	for rows.Next() {
 		var username string