Optimize queries on footprint attributes

Entire-Checkpoint: 81bf5e62e311

internal/repository/jobQuery.go

@@ -63,12 +63,11 @@ func (r *JobRepository) QueryJobs(
 			}
 		} else {
 			// Order by footprint JSON field values
-			query = query.Where("JSON_VALID(meta_data)")
 			switch order.Order {
 			case model.SortDirectionEnumAsc:
-				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") ASC", field))
+				query = query.OrderBy(fmt.Sprintf("json_extract(footprint, '$.%s') ASC", field))
 			case model.SortDirectionEnumDesc:
-				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") DESC", field))
+				query = query.OrderBy(fmt.Sprintf("json_extract(footprint, '$.%s') DESC", field))
 			default:
 				return nil, errors.New("invalid sorting order for footprint")
 			}
@@ -276,28 +275,26 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select
 	return query
 }
 
-// buildIntCondition creates a BETWEEN clause for integer range filters.
+// buildIntCondition creates clauses for integer range filters, using BETWEEN only if required.
-// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
 func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuilder) sq.SelectBuilder {
-	if cond.From != 0 && cond.To != 0 {
+	if cond.From != 1 && cond.To != 0 {
 		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
-	} else if cond.From != 0 {
+	} else if cond.From != 1 && cond.To == 0 {
 		return query.Where(field+" >= ?", cond.From)
-	} else if cond.To != 0 {
+	} else if cond.From == 1 && cond.To != 0 {
 		return query.Where(field+" <= ?", cond.To)
 	} else {
 		return query
 	}
 }
 
-// buildFloatCondition creates a BETWEEN clause for float range filters.
+// buildFloatCondition creates a clauses for float range filters, using BETWEEN only if required.
-// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
 func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
-	if cond.From != 0.0 && cond.To != 0.0 {
+	if cond.From != 1.0 && cond.To != 0.0 {
 		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
-	} else if cond.From != 0.0 {
+	} else if cond.From != 1.0 && cond.To == 0.0 {
 		return query.Where(field+" >= ?", cond.From)
-	} else if cond.To != 0.0 {
+	} else if cond.From == 1.0 && cond.To != 0.0 {
 		return query.Where(field+" <= ?", cond.To)
 	} else {
 		return query
@@ -336,16 +333,14 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui
 	}
 }
 
-// buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column.
+// buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column, using BETWEEN only if required.
-// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
+func buildFloatJSONCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
-func buildFloatJSONCondition(condName string, condRange *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
+	if cond.From != 1.0 && cond.To != 0.0 {
-	query = query.Where("JSON_VALID(footprint)")
+		return query.Where("json_extract(footprint, '$."+field+"') BETWEEN ? AND ?", cond.From, cond.To)
-	if condRange.From != 0.0 && condRange.To != 0.0 {
+	} else if cond.From != 1.0 && cond.To == 0.0 {
-		return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") BETWEEN ? AND ?", condRange.From, condRange.To)
+		return query.Where("json_extract(footprint, '$."+field+"') >= ?", cond.From)
-	} else if condRange.From != 0.0 {
+	} else if cond.From == 1.0 && cond.To != 0.0 {
-		return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") >= ?", condRange.From)
+		return query.Where("json_extract(footprint, '$."+field+"') <= ?", cond.To)
-	} else if condRange.To != 0.0 {
-		return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") <= ?", condRange.To)
 	} else {
 		return query
 	}

internal/repository/migration.go

@@ -21,10 +21,11 @@ import (
 // is added to internal/repository/migrations/sqlite3/.
 //
 // Version history:
-//   - Version 10: Current version
+//   - Version 11: Add expression indexes on footprint JSON fields
+//   - Version 10: Previous version
 //
 // Migration files are embedded at build time from the migrations directory.
-const Version uint = 10
+const Version uint = 11
 
 //go:embed migrations/*
 var migrationFiles embed.FS

internal/repository/migrations/sqlite3/11_add-footprint-indexes.down.sql

@@ -0,0 +1,15 @@
+-- Drop standalone expression indexes
+DROP INDEX IF EXISTS jobs_fp_flops_any_avg;
+DROP INDEX IF EXISTS jobs_fp_mem_bw_avg;
+DROP INDEX IF EXISTS jobs_fp_mem_used_max;
+DROP INDEX IF EXISTS jobs_fp_cpu_load_avg;
+DROP INDEX IF EXISTS jobs_fp_net_bw_avg;
+DROP INDEX IF EXISTS jobs_fp_net_data_vol_total;
+DROP INDEX IF EXISTS jobs_fp_file_bw_avg;
+DROP INDEX IF EXISTS jobs_fp_file_data_vol_total;
+
+-- Drop composite indexes
+DROP INDEX IF EXISTS jobs_cluster_fp_cpu_load_avg;
+DROP INDEX IF EXISTS jobs_cluster_fp_flops_any_avg;
+DROP INDEX IF EXISTS jobs_cluster_fp_mem_bw_avg;
+DROP INDEX IF EXISTS jobs_cluster_fp_mem_used_max;

internal/repository/migrations/sqlite3/11_add-footprint-indexes.up.sql

@@ -0,0 +1,19 @@
+-- Expression indexes on footprint JSON fields for WHERE and ORDER BY optimization.
+-- SQLite matches expressions textually, so queries must use exactly:
+--   json_extract(footprint, '$.field')
+
+-- Standalone expression indexes (for filtering and sorting)
+CREATE INDEX IF NOT EXISTS jobs_fp_flops_any_avg ON job (json_extract(footprint, '$.flops_any_avg'));
+CREATE INDEX IF NOT EXISTS jobs_fp_mem_bw_avg ON job (json_extract(footprint, '$.mem_bw_avg'));
+CREATE INDEX IF NOT EXISTS jobs_fp_mem_used_max ON job (json_extract(footprint, '$.mem_used_max'));
+CREATE INDEX IF NOT EXISTS jobs_fp_cpu_load_avg ON job (json_extract(footprint, '$.cpu_load_avg'));
+CREATE INDEX IF NOT EXISTS jobs_fp_net_bw_avg ON job (json_extract(footprint, '$.net_bw_avg'));
+CREATE INDEX IF NOT EXISTS jobs_fp_net_data_vol_total ON job (json_extract(footprint, '$.net_data_vol_total'));
+CREATE INDEX IF NOT EXISTS jobs_fp_file_bw_avg ON job (json_extract(footprint, '$.file_bw_avg'));
+CREATE INDEX IF NOT EXISTS jobs_fp_file_data_vol_total ON job (json_extract(footprint, '$.file_data_vol_total'));
+
+-- Composite indexes with cluster (for common filter+sort combinations)
+CREATE INDEX IF NOT EXISTS jobs_cluster_fp_cpu_load_avg ON job (cluster, json_extract(footprint, '$.cpu_load_avg'));
+CREATE INDEX IF NOT EXISTS jobs_cluster_fp_flops_any_avg ON job (cluster, json_extract(footprint, '$.flops_any_avg'));
+CREATE INDEX IF NOT EXISTS jobs_cluster_fp_mem_bw_avg ON job (cluster, json_extract(footprint, '$.mem_bw_avg'));
+CREATE INDEX IF NOT EXISTS jobs_cluster_fp_mem_used_max ON job (cluster, json_extract(footprint, '$.mem_used_max'));

internal/repository/stats.go

@@ -921,16 +921,14 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 	// Special case: value == peak would create bin N+1, so we test for equality
 	// and multiply peak by 0.999999999 to force it into bin N.
 	binQuery := fmt.Sprintf(`CAST(
-		((case when json_extract(footprint, "$.%s") = %f then %f*0.999999999 else json_extract(footprint, "$.%s") end) / %f)
+		((case when json_extract(footprint, '$.%s') = %f then %f*0.999999999 else json_extract(footprint, '$.%s') end) / %f)
 		* %v as INTEGER )`,
 		(metric + "_" + footprintStat), peak, peak, (metric + "_" + footprintStat), peak, *bins)
 
 	mainQuery := sq.Select(
 		fmt.Sprintf(`%s + 1 as bin`, binQuery),
 		`count(*) as count`,
-	).From("job").Where(
+	).From("job").Where(fmt.Sprintf(`json_extract(footprint, '$.%s') is not null and json_extract(footprint, '$.%s') <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak))
-		"JSON_VALID(footprint)",
-	).Where(fmt.Sprintf(`json_extract(footprint, "$.%s") is not null and json_extract(footprint, "$.%s") <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak))
 
 	mainQuery, qerr := SecurityCheck(ctx, mainQuery)
 	if qerr != nil {

web/frontend/src/generic/filters/Energy.svelte

@@ -29,7 +29,7 @@
 
   /* Const */
   const minEnergyPreset = 1;
-  const maxEnergyPreset = 1000;
+  const maxEnergyPreset = 100;
 
   /* Derived */
   // Pending