Merge branch 'dev' of github.com:ClusterCockpit/cc-backend into dev

internal/repository/jobFind.go

@@ -280,6 +280,7 @@ func (r *JobRepository) FindConcurrentJobs(
 	stopTimeTail := stopTime - overlapBufferEnd
 	startTimeFront := startTime + overlapBufferEnd
 
+	// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Can this be optimized?
 	queryRunning := query.Where("job.job_state = ?").Where("(job.start_time BETWEEN ? AND ? OR job.start_time < ?)",
 		"running", startTimeTail, stopTimeTail, startTime)
 	// Get At Least One Exact Hostname Match from JSON Resources Array in Database

internal/repository/jobQuery.go

@@ -274,17 +274,36 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select
 }
 
 // buildIntCondition creates a BETWEEN clause for integer range filters.
+// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
 func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuilder) sq.SelectBuilder {
-	return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+	if cond.From != 0 && cond.To != 0 {
+		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+	} else if cond.From != 0 {
+		return query.Where("? <= "+field, cond.From)
+	} else if cond.To != 0 {
+		return query.Where(field+" <= ?", cond.To)
+	} else {
+		return query
+	}
 }
 
 // buildFloatCondition creates a BETWEEN clause for float range filters.
+// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
 func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
-	return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+	if cond.From != 0.0 && cond.To != 0.0 {
+		return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+	} else if cond.From != 0.0 {
+		return query.Where("? <= "+field, cond.From)
+	} else if cond.To != 0.0 {
+		return query.Where(field+" <= ?", cond.To)
+	} else {
+		return query
+	}
 }
 
 // buildTimeCondition creates time range filters supporting absolute timestamps,
 // relative time ranges (last6h, last24h, last7d, last30d), or open-ended ranges.
+// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
 func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBuilder) sq.SelectBuilder {
 	if cond.From != nil && cond.To != nil {
 		return query.Where(field+" BETWEEN ? AND ?", cond.From.Unix(), cond.To.Unix())
@@ -308,16 +327,25 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui
 			cclog.Debugf("No known named timeRange: startTime.range = %s", cond.Range)
 			return query
 		}
-		return query.Where(field+" BETWEEN ? AND ?", then, now)
+		return query.Where("? <= "+field, then)
 	} else {
 		return query
 	}
 }
 
 // buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column.
+// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required
 func buildFloatJSONCondition(condName string, condRange *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
 	query = query.Where("JSON_VALID(footprint)")
-	return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") BETWEEN ? AND ?", condRange.From, condRange.To)
+	if condRange.From != 0.0 && condRange.To != 0.0 {
+		return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") BETWEEN ? AND ?", condRange.From, condRange.To)
+	} else if condRange.From != 0.0 {
+		return query.Where("? <= JSON_EXTRACT(footprint, \"$."+condName+"\")", condRange.From)
+	} else if condRange.To != 0.0 {
+		return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") <= ?", condRange.To)
+	} else {
+		return query
+	}
 }
 
 // buildStringCondition creates filters for string fields supporting equality,

internal/repository/migrations/sqlite3/09_add-job-cache.up.sql

@@ -124,13 +124,15 @@ CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project);
 CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster);
 -- Cluster Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_cluster_starttime ON job (cluster, start_time);
-CREATE INDEX IF NOT EXISTS jobs_cluster_duration ON job (cluster, duration);
 CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_cluster_numhwthreads ON job (cluster, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_cluster_numacc ON job (cluster, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_cluster_energy ON job (cluster, energy);
 
+-- Cluster Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_duration_starttime ON job (cluster, duration, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_starttime_duration ON job (cluster, start_time, duration);
+
 -- Cluster+Partition Filter
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_user ON job (cluster, cluster_partition, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_project ON job (cluster, cluster_partition, project);
@@ -138,76 +140,90 @@ CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, clus
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_shared ON job (cluster, cluster_partition, shared);
 
 -- Cluster+Partition Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, cluster_partition, start_time);
-CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, cluster_partition, duration);
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy);
 
+-- Cluster+Partition Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration_starttime ON job (cluster, cluster_partition, duration, start_time);
+CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime_duration ON job (cluster, cluster_partition, start_time, duration);
+
 -- Cluster+JobState Filter
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project);
 -- Cluster+JobState Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time);
-CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration ON job (cluster, job_state, duration);
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numhwthreads ON job (cluster, job_state, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_state, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy);
 
+-- Cluster+JobState Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime_duration ON job (cluster, job_state, start_time, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration_starttime ON job (cluster, job_state, duration, start_time);
+
 -- Cluster+Shared Filter
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_user ON job (cluster, shared, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_project ON job (cluster, shared, project);
 -- Cluster+Shared Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_cluster_shared_starttime ON job (cluster, shared, start_time);
-CREATE INDEX IF NOT EXISTS jobs_cluster_shared_duration ON job (cluster, shared, duration);
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numnodes ON job (cluster, shared, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numhwthreads ON job (cluster, shared, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numacc ON job (cluster, shared, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_cluster_shared_energy ON job (cluster, shared, energy);
 
+-- Cluster+Shared Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_cluster_shared_starttime_duration ON job (cluster, shared, start_time, duration);
+CREATE INDEX IF NOT EXISTS jobs_cluster_shared_duration_starttime ON job (cluster, shared, duration, start_time);
+
 -- User Filter
 -- User Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time);
-CREATE INDEX IF NOT EXISTS jobs_user_duration ON job (hpc_user, duration);
 CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_user_numhwthreads ON job (hpc_user, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (hpc_user, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy);
 
+-- Cluster+Shared Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_user_starttime_duration ON job (hpc_user, start_time, duration);
+CREATE INDEX IF NOT EXISTS jobs_user_duration_starttime ON job (hpc_user, duration, start_time);
+
 -- Project Filter
 CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user);
 -- Project Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time);
-CREATE INDEX IF NOT EXISTS jobs_project_duration ON job (project, duration);
 CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_project_numhwthreads ON job (project, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_project_numacc ON job (project, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy);
 
+-- Cluster+Shared Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_project_starttime_duration ON job (project, start_time, duration);
+CREATE INDEX IF NOT EXISTS jobs_project_duration_starttime ON job (project, duration, start_time);
+
 -- JobState Filter
 CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project);
 -- JobState Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime ON job (job_state, start_time);
-CREATE INDEX IF NOT EXISTS jobs_jobstate_duration ON job (job_state, duration);
 CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_jobstate_numhwthreads ON job (job_state, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_jobstate_numacc ON job (job_state, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_jobstate_energy ON job (job_state, energy);
 
+-- Cluster+Shared Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime_duration ON job (job_state, start_time, duration);
+CREATE INDEX IF NOT EXISTS jobs_jobstate_duration_starttime ON job (job_state, duration, start_time);
+
 -- Shared Filter
 CREATE INDEX IF NOT EXISTS jobs_shared_user ON job (shared, hpc_user);
 CREATE INDEX IF NOT EXISTS jobs_shared_project ON job (shared, project);
 -- Shared Filter Sorting
-CREATE INDEX IF NOT EXISTS jobs_shared_starttime ON job (shared, start_time);
-CREATE INDEX IF NOT EXISTS jobs_shared_duration ON job (shared, duration);
 CREATE INDEX IF NOT EXISTS jobs_shared_numnodes ON job (shared, num_nodes);
 CREATE INDEX IF NOT EXISTS jobs_shared_numhwthreads ON job (shared, num_hwthreads);
 CREATE INDEX IF NOT EXISTS jobs_shared_numacc ON job (shared, num_acc);
 CREATE INDEX IF NOT EXISTS jobs_shared_energy ON job (shared, energy);
 
+-- Cluster+Shared Time Filter Sorting
+CREATE INDEX IF NOT EXISTS jobs_shared_starttime_duration ON job (shared, start_time, duration);
+CREATE INDEX IF NOT EXISTS jobs_shared_duration_starttime ON job (shared, duration, start_time);
+
 -- ArrayJob Filter
 CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time);
 CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time);
@@ -226,6 +242,10 @@ CREATE INDEX IF NOT EXISTS jobs_numhwthreads_duration ON job (num_hwthreads, dur
 CREATE INDEX IF NOT EXISTS jobs_numacc_duration ON job (num_acc, duration);
 CREATE INDEX IF NOT EXISTS jobs_energy_duration ON job (energy, duration);
 
+-- Backup Indices For High Variety Columns
+CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time);
+CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration);
+
 -- Notes:
 -- Cluster+Partition+Jobstate Filter: Tested -> Full Array Of Combinations non-required
 -- Cluster+JobState+Shared Filter: Tested -> No further timing improvement

internal/routerConfig/routes.go

@@ -277,10 +277,22 @@ func buildFilterPresets(query url.Values) map[string]interface{} {
 	if query.Get("duration") != "" {
 		parts := strings.Split(query.Get("duration"), "-")
 		if len(parts) == 2 {
-			a, e1 := strconv.Atoi(parts[0])
+			if parts[0] == "lessthan" {
-			b, e2 := strconv.Atoi(parts[1])
+				lt, lte := strconv.Atoi(parts[1])
-			if e1 == nil && e2 == nil {
+				if lte == nil {
-				filterPresets["duration"] = map[string]int{"from": a, "to": b}
+					filterPresets["duration"] = map[string]int{"lessThan": lt, "from": 0, "to": 0}
+				}
+			} else if parts[0] == "morethan" {
+				mt, mte := strconv.Atoi(parts[1])
+				if mte == nil {
+					filterPresets["duration"] = map[string]int{"moreThan": mt, "from": 0, "to": 0}
+				}
+			} else {
+				a, e1 := strconv.Atoi(parts[0])
+				b, e2 := strconv.Atoi(parts[1])
+				if e1 == nil && e2 == nil {
+					filterPresets["duration"] = map[string]int{"from": a, "to": b}
+				}
 			}
 		}
 	}

web/frontend/src/generic/Filters.svelte

@@ -192,14 +192,14 @@
       items.push({
         startTime: { range: filters.startTime.range },
       });
-    if (filters.duration.from || filters.duration.to)
+    if (filters.duration.from && filters.duration.to)
       items.push({
         duration: { from: filters.duration.from, to: filters.duration.to },
       });
     if (filters.duration.lessThan)
-      items.push({ duration: { from: 0, to: filters.duration.lessThan } });
+      items.push({ duration: { to: filters.duration.lessThan, from: 0 } });
     if (filters.duration.moreThan)
-      items.push({ duration: { from: filters.duration.moreThan, to: 604800 } }); // 7 days to include special jobs with long runtimes
+      items.push({ duration: { to: 0, from: filters.duration.moreThan } });
     if (filters.energy.from || filters.energy.to)
       items.push({
         energy: { from: filters.energy.from, to: filters.energy.to },
@@ -266,9 +266,9 @@
     if (filters.duration.from && filters.duration.to)
       opts.push(`duration=${filters.duration.from}-${filters.duration.to}`);
     if (filters.duration.lessThan)
-      opts.push(`duration=0-${filters.duration.lessThan}`);
+      opts.push(`duration=lessthan-${filters.duration.lessThan}`);
     if (filters.duration.moreThan)
-      opts.push(`duration=${filters.duration.moreThan}-604800`);
+      opts.push(`duration=morethan-${filters.duration.moreThan}`);
     if (filters.tags.length != 0)
       for (let tag of filters.tags) opts.push(`tag=${tag}`);
     if (filters.numNodes.from && filters.numNodes.to)

web/frontend/src/generic/filters/Duration.svelte

@@ -31,14 +31,16 @@
     setFilter
   } = $props();
 
+  /* States */
+  let lessState = $state({ hours:0, mins:0 });
+  let moreState = $state({ hours:0, mins:0 });
+  let fromState = $state({ hours:0, mins:0 });
+  let toState = $state({ hours:0, mins:0 });
+
   /* Derived */
   let pendingDuration = $derived(presetDuration);
-  let lessState = $derived(secsToHoursAndMins(presetDuration?.lessThan));
-  let moreState = $derived(secsToHoursAndMins(presetDuration?.moreThan));
-  let fromState = $derived(secsToHoursAndMins(presetDuration?.from));
-  let toState = $derived(secsToHoursAndMins(presetDuration?.to));
 
-  const lessDisabled = $derived(
+  let lessDisabled = $derived(
     moreState.hours !== 0 ||
     moreState.mins !== 0 ||
     fromState.hours !== 0 ||
@@ -47,7 +49,7 @@
     toState.mins !== 0
   );
 
-  const moreDisabled = $derived(
+  let moreDisabled = $derived(
     lessState.hours !== 0 ||
     lessState.mins !== 0 ||
     fromState.hours !== 0 ||
@@ -56,13 +58,27 @@
     toState.mins !== 0
   );
 
-  const betweenDisabled = $derived(
+  let betweenDisabled = $derived(
     moreState.hours !== 0 ||
     moreState.mins !== 0 ||
     lessState.hours !== 0 ||
     lessState.mins !== 0
   )
 
+  /* Effects */
+  $effect(() => {
+    lessState = secsToHoursAndMins(pendingDuration?.lessThan);
+  });
+  $effect(() => {
+    moreState = secsToHoursAndMins(pendingDuration?.moreThan);
+  });
+  $effect(() => {
+    fromState = secsToHoursAndMins(pendingDuration?.from);
+  });
+  $effect(() => {
+    toState = secsToHoursAndMins(pendingDuration?.to);
+  });
+
   /* Functions */
   function resetPending() {
     pendingDuration = {

web/frontend/src/systems/nodelist/NodeListRow.svelte

@@ -116,7 +116,7 @@
 
       pendingExtendedLegendData = {};
       for (const accId of accSet) {
-        const matchJob = $nodeJobsData.data.jobs.items.find((i) => i.resources.find((r) => r.accelerators.includes(accId)))
+        const matchJob = $nodeJobsData?.data?.jobs?.items?.find((i) => i?.resources?.find((r) => r?.accelerators?.includes(accId))) || null
         const matchUser = matchJob?.user ? matchJob.user : null
         pendingExtendedLegendData[accId] = {
           user: (scrambleNames && matchUser)