Review duration filter handling, update migration indices

internal/repository/jobFind.go

@@ -280,6 +280,7 @@ func (r *JobRepository) FindConcurrentJobs(
 	stopTimeTail := stopTime - overlapBufferEnd
 	startTimeFront := startTime + overlapBufferEnd
 
+	// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Can this be optimized?
 	queryRunning := query.Where("job.job_state = ?").Where("(job.start_time BETWEEN ? AND ? OR job.start_time < ?)",
 		"running", startTimeTail, stopTimeTail, startTime)
 	// Get At Least One Exact Hostname Match from JSON Resources Array in Database

internal/repository/jobQuery.go

@@ -274,17 +274,36 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select
 }
 
 // buildIntCondition creates a BETWEEN clause for integer range filters.
+// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
 func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuilder) sq.SelectBuilder {
-	return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+	if cond.From != 0 && cond.To != 0 {
+		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+	} else if cond.From != 0 {
+		return query.Where("? <= "+field, cond.From)
+	} else if cond.To != 0 {
+		return query.Where(field+" <= ?", cond.To)
+	} else {
+		return query
+	}
 }
 
 // buildFloatCondition creates a BETWEEN clause for float range filters.
+// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
 func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
-	return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+	if cond.From != 0.0 && cond.To != 0.0 {
+		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+	} else if cond.From != 0.0 {
+		return query.Where("? <= "+field, cond.From)
+	} else if cond.To != 0.0 {
+		return query.Where(field+" <= ?", cond.To)
+	} else {
+		return query
+	}
 }
 
 // buildTimeCondition creates time range filters supporting absolute timestamps,
 // relative time ranges (last6h, last24h, last7d, last30d), or open-ended ranges.
+// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
 func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBuilder) sq.SelectBuilder {
 	if cond.From != nil && cond.To != nil {
 		return query.Where(field+" BETWEEN ? AND ?", cond.From.Unix(), cond.To.Unix())
@@ -308,16 +327,25 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui
 			cclog.Debugf("No known named timeRange: startTime.range = %s", cond.Range)
 			return query
 		}
-		return query.Where(field+" BETWEEN ? AND ?", then, now)
+		return query.Where("? <= "+field, then)
 	} else {
 		return query
 	}
 }
 
 // buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column.
+// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
 func buildFloatJSONCondition(condName string, condRange *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
 	query = query.Where("JSON_VALID(footprint)")
-	return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") BETWEEN ? AND ?", condRange.From, condRange.To)
+	if condRange.From != 0.0 && condRange.To != 0.0 {
+		return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") BETWEEN ? AND ?", condRange.From, condRange.To)
+	} else if condRange.From != 0.0 {
+		return query.Where("? <= JSON_EXTRACT(footprint, \"$."+condName+"\")", condRange.From)
+	} else if condRange.To != 0.0 {
+		return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") <= ?", condRange.To)
+	} else {
+		return query
+	}
 }
 
 // buildStringCondition creates filters for string fields supporting equality,

internal/repository/migrations/sqlite3/09_add-job-cache.up.sql

@@ -124,13 +124,15 @@ CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project);
 CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster);
 -- Cluster Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_cluster_starttime ON job (cluster, start_time);
-CREATE INDEX IF NOT EXISTS jobs_cluster_duration ON job (cluster, duration);
 CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_cluster_numhwthreads ON job (cluster, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_cluster_numacc ON job (cluster, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_cluster_energy ON job (cluster, energy);
 
+-- Cluster Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_duration_starttime ON job (cluster, duration, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_starttime_duration ON job (cluster, start_time, duration);
+
 -- Cluster+Partition Filter
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_user ON job (cluster, cluster_partition, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_project ON job (cluster, cluster_partition, project);
@@ -138,35 +140,41 @@ CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, clus
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_shared ON job (cluster, cluster_partition, shared);
 
 -- Cluster+Partition Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, cluster_partition, start_time);
-CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, cluster_partition, duration);
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy);
 
+-- Cluster+Partition Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration_starttime ON job (cluster, cluster_partition, duration, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime_duration ON job (cluster, cluster_partition, start_time, duration);
+
 -- Cluster+JobState Filter
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project);
 -- Cluster+JobState Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time);
-CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration ON job (cluster, job_state, duration);
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numhwthreads ON job (cluster, job_state, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_state, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy);
 
+-- Cluster+JobState Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime_duration ON job (cluster, job_state, start_time, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration_starttime ON job (cluster, job_state, duration, start_time);
+
 -- Cluster+Shared Filter
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_user ON job (cluster, shared, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_project ON job (cluster, shared, project);
 -- Cluster+Shared Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_cluster_shared_starttime ON job (cluster, shared, start_time);
-CREATE INDEX IF NOT EXISTS jobs_cluster_shared_duration ON job (cluster, shared, duration);
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numnodes ON job (cluster, shared, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numhwthreads ON job (cluster, shared, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numacc ON job (cluster, shared, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_energy ON job (cluster, shared, energy);
 
+-- Cluster+Shared Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_shared_starttime_duration ON job (cluster, shared, start_time, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_shared_duration_starttime ON job (cluster, shared, duration, start_time);
+
 -- User Filter
 -- User Filter Sorting
 CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time);
@@ -179,35 +187,41 @@ CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy);
 -- Project Filter
 CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user);
 -- Project Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time);
-CREATE INDEX IF NOT EXISTS jobs_project_duration ON job (project, duration);
 CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_project_numhwthreads ON job (project, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_project_numacc ON job (project, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy);
 
+-- Cluster+Shared Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_project_starttime_duration ON job (project, start_time, duration);
+CREATE INDEX IF NOT EXISTS jobs_project_duration_starttime ON job (project, duration, start_time);
+
 -- JobState Filter
 CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project);
 -- JobState Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime ON job (job_state, start_time);
-CREATE INDEX IF NOT EXISTS jobs_jobstate_duration ON job (job_state, duration);
 CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_jobstate_numhwthreads ON job (job_state, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_jobstate_numacc ON job (job_state, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_jobstate_energy ON job (job_state, energy);
 
+-- Cluster+Shared Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime_duration ON job (job_state, start_time, duration);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_duration_starttime ON job (job_state, duration, start_time);
+
 -- Shared Filter
 CREATE INDEX IF NOT EXISTS jobs_shared_user ON job (shared, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_shared_project ON job (shared, project);
 -- Shared Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_shared_starttime ON job (shared, start_time);
-CREATE INDEX IF NOT EXISTS jobs_shared_duration ON job (shared, duration);
 CREATE INDEX IF NOT EXISTS jobs_shared_numnodes ON job (shared, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_shared_numhwthreads ON job (shared, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_shared_numacc ON job (shared, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_shared_energy ON job (shared, energy);
 
+-- Cluster+Shared Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_shared_starttime_duration ON job (shared, start_time, duration);
+CREATE INDEX IF NOT EXISTS jobs_shared_duration_starttime ON job (shared, duration, start_time);
+
 -- ArrayJob Filter
 CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time);
 CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time);
@@ -226,6 +240,10 @@ CREATE INDEX IF NOT EXISTS jobs_numhwthreads_duration ON job (num_hwthreads, dur
 CREATE INDEX IF NOT EXISTS jobs_numacc_duration ON job (num_acc, duration);
 CREATE INDEX IF NOT EXISTS jobs_energy_duration ON job (energy, duration);
 
+-- Backup Indices For High Variety Columns
+CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time);
+CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration);
+
 -- Notes:
 -- Cluster+Partition+Jobstate Filter: Tested -> Full Array Of Combinations non-required
 -- Cluster+JobState+Shared Filter: Tested -> No further timing improvement