Optimize queries on footprint attributes

Entire-Checkpoint: 81bf5e62e311

internal/repository/jobQuery.go

@@ -63,12 +63,11 @@ func (r *JobRepository) QueryJobs(
 			}
 		} else {
 			// Order by footprint JSON field values
-			query = query.Where("JSON_VALID(meta_data)")
 			switch order.Order {
 			case model.SortDirectionEnumAsc:
-				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") ASC", field))
+				query = query.OrderBy(fmt.Sprintf("json_extract(footprint, '$.%s') ASC", field))
 			case model.SortDirectionEnumDesc:
-				query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") DESC", field))
+				query = query.OrderBy(fmt.Sprintf("json_extract(footprint, '$.%s') DESC", field))
 			default:
 				return nil, errors.New("invalid sorting order for footprint")
 			}
@@ -336,13 +335,12 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui
 
 // buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column, using BETWEEN only if required.
 func buildFloatJSONCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
-	query = query.Where("JSON_VALID(footprint)")
 	if cond.From != 1.0 && cond.To != 0.0 {
-		return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") BETWEEN ? AND ?", cond.From, cond.To)
+		return query.Where("json_extract(footprint, '$."+field+"') BETWEEN ? AND ?", cond.From, cond.To)
 	} else if cond.From != 1.0 && cond.To == 0.0 {
-		return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") >= ?", cond.From)
+		return query.Where("json_extract(footprint, '$."+field+"') >= ?", cond.From)
 	} else if cond.From == 1.0 && cond.To != 0.0 {
-		return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") <= ?", cond.To)
+		return query.Where("json_extract(footprint, '$."+field+"') <= ?", cond.To)
 	} else {
 		return query
 	}

internal/repository/migration.go

@@ -21,10 +21,11 @@ import (
 // is added to internal/repository/migrations/sqlite3/.
 //
 // Version history:
-//   - Version 10: Current version
+//   - Version 11: Add expression indexes on footprint JSON fields
+//   - Version 10: Previous version
 //
 // Migration files are embedded at build time from the migrations directory.
-const Version uint = 10
+const Version uint = 11
 
 //go:embed migrations/*
 var migrationFiles embed.FS

internal/repository/migrations/sqlite3/11_add-footprint-indexes.down.sql

@@ -0,0 +1,15 @@
+-- Drop standalone expression indexes
+DROP INDEX IF EXISTS jobs_fp_flops_any_avg;
+DROP INDEX IF EXISTS jobs_fp_mem_bw_avg;
+DROP INDEX IF EXISTS jobs_fp_mem_used_max;
+DROP INDEX IF EXISTS jobs_fp_cpu_load_avg;
+DROP INDEX IF EXISTS jobs_fp_net_bw_avg;
+DROP INDEX IF EXISTS jobs_fp_net_data_vol_total;
+DROP INDEX IF EXISTS jobs_fp_file_bw_avg;
+DROP INDEX IF EXISTS jobs_fp_file_data_vol_total;
+
+-- Drop composite indexes
+DROP INDEX IF EXISTS jobs_cluster_fp_cpu_load_avg;
+DROP INDEX IF EXISTS jobs_cluster_fp_flops_any_avg;
+DROP INDEX IF EXISTS jobs_cluster_fp_mem_bw_avg;
+DROP INDEX IF EXISTS jobs_cluster_fp_mem_used_max;

internal/repository/migrations/sqlite3/11_add-footprint-indexes.up.sql

@@ -0,0 +1,19 @@
+-- Expression indexes on footprint JSON fields for WHERE and ORDER BY optimization.
+-- SQLite matches expressions textually, so queries must use exactly:
+--   json_extract(footprint, '$.field')
+
+-- Standalone expression indexes (for filtering and sorting)
+CREATE INDEX IF NOT EXISTS jobs_fp_flops_any_avg ON job (json_extract(footprint, '$.flops_any_avg'));
+CREATE INDEX IF NOT EXISTS jobs_fp_mem_bw_avg ON job (json_extract(footprint, '$.mem_bw_avg'));
+CREATE INDEX IF NOT EXISTS jobs_fp_mem_used_max ON job (json_extract(footprint, '$.mem_used_max'));
+CREATE INDEX IF NOT EXISTS jobs_fp_cpu_load_avg ON job (json_extract(footprint, '$.cpu_load_avg'));
+CREATE INDEX IF NOT EXISTS jobs_fp_net_bw_avg ON job (json_extract(footprint, '$.net_bw_avg'));
+CREATE INDEX IF NOT EXISTS jobs_fp_net_data_vol_total ON job (json_extract(footprint, '$.net_data_vol_total'));
+CREATE INDEX IF NOT EXISTS jobs_fp_file_bw_avg ON job (json_extract(footprint, '$.file_bw_avg'));
+CREATE INDEX IF NOT EXISTS jobs_fp_file_data_vol_total ON job (json_extract(footprint, '$.file_data_vol_total'));
+
+-- Composite indexes with cluster (for common filter+sort combinations)
+CREATE INDEX IF NOT EXISTS jobs_cluster_fp_cpu_load_avg ON job (cluster, json_extract(footprint, '$.cpu_load_avg'));
+CREATE INDEX IF NOT EXISTS jobs_cluster_fp_flops_any_avg ON job (cluster, json_extract(footprint, '$.flops_any_avg'));
+CREATE INDEX IF NOT EXISTS jobs_cluster_fp_mem_bw_avg ON job (cluster, json_extract(footprint, '$.mem_bw_avg'));
+CREATE INDEX IF NOT EXISTS jobs_cluster_fp_mem_used_max ON job (cluster, json_extract(footprint, '$.mem_used_max'));

internal/repository/stats.go

@@ -921,16 +921,14 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
 	// Special case: value == peak would create bin N+1, so we test for equality
 	// and multiply peak by 0.999999999 to force it into bin N.
 	binQuery := fmt.Sprintf(`CAST(
-		((case when json_extract(footprint, "$.%s") = %f then %f*0.999999999 else json_extract(footprint, "$.%s") end) / %f)
+		((case when json_extract(footprint, '$.%s') = %f then %f*0.999999999 else json_extract(footprint, '$.%s') end) / %f)
 		* %v as INTEGER )`,
 		(metric + "_" + footprintStat), peak, peak, (metric + "_" + footprintStat), peak, *bins)
 
 	mainQuery := sq.Select(
 		fmt.Sprintf(`%s + 1 as bin`, binQuery),
 		`count(*) as count`,
-	).From("job").Where(
+	).From("job").Where(fmt.Sprintf(`json_extract(footprint, '$.%s') is not null and json_extract(footprint, '$.%s') <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak))
-		"JSON_VALID(footprint)",
-	).Where(fmt.Sprintf(`json_extract(footprint, "$.%s") is not null and json_extract(footprint, "$.%s") <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak))
 
 	mainQuery, qerr := SecurityCheck(ctx, mainQuery)
 	if qerr != nil {