From 836e6e4242c78538390266df90f9bfbeb5e9b79d Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 26 Jan 2026 15:53:00 +0100 Subject: [PATCH 1/3] Review duration filter handling, update migration indices --- internal/repository/jobFind.go | 1 + internal/repository/jobQuery.go | 36 +++++++++++++-- .../sqlite3/09_add-job-cache.up.sql | 46 +++++++++++++------ internal/routerConfig/routes.go | 20 ++++++-- web/frontend/src/generic/Filters.svelte | 10 ++-- .../src/generic/filters/Duration.svelte | 30 +++++++++--- 6 files changed, 109 insertions(+), 34 deletions(-) diff --git a/internal/repository/jobFind.go b/internal/repository/jobFind.go index 4386be2d..d79847a0 100644 --- a/internal/repository/jobFind.go +++ b/internal/repository/jobFind.go @@ -280,6 +280,7 @@ func (r *JobRepository) FindConcurrentJobs( stopTimeTail := stopTime - overlapBufferEnd startTimeFront := startTime + overlapBufferEnd + // Reminder: BETWEEN Queries are slower and dont use indices as frequently: Can this be optimized? queryRunning := query.Where("job.job_state = ?").Where("(job.start_time BETWEEN ? AND ? OR job.start_time < ?)", "running", startTimeTail, stopTimeTail, startTime) // Get At Least One Exact Hostname Match from JSON Resources Array in Database diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index 745fa32d..cf7010ee 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -274,17 +274,36 @@ func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.Select } // buildIntCondition creates a BETWEEN clause for integer range filters. +// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required func buildIntCondition(field string, cond *config.IntRange, query sq.SelectBuilder) sq.SelectBuilder { - return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To) + if cond.From != 0 && cond.To != 0 { + return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To) + } else if cond.From != 0 { + return query.Where("? <= "+field, cond.From) + } else if cond.To != 0 { + return query.Where(field+" <= ?", cond.To) + } else { + return query + } } // buildFloatCondition creates a BETWEEN clause for float range filters. +// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder { - return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To) + if cond.From != 0.0 && cond.To != 0.0 { + return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To) + } else if cond.From != 0.0 { + return query.Where("? <= "+field, cond.From) + } else if cond.To != 0.0 { + return query.Where(field+" <= ?", cond.To) + } else { + return query + } } // buildTimeCondition creates time range filters supporting absolute timestamps, // relative time ranges (last6h, last24h, last7d, last30d), or open-ended ranges. +// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBuilder) sq.SelectBuilder { if cond.From != nil && cond.To != nil { return query.Where(field+" BETWEEN ? AND ?", cond.From.Unix(), cond.To.Unix()) @@ -308,16 +327,25 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui cclog.Debugf("No known named timeRange: startTime.range = %s", cond.Range) return query } - return query.Where(field+" BETWEEN ? AND ?", then, now) + return query.Where("? <= "+field, then) } else { return query } } // buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column. +// Reminder: BETWEEN Queries are slower and dont use indices as frequently: Only use if both conditions required func buildFloatJSONCondition(condName string, condRange *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder { query = query.Where("JSON_VALID(footprint)") - return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") BETWEEN ? AND ?", condRange.From, condRange.To) + if condRange.From != 0.0 && condRange.To != 0.0 { + return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") BETWEEN ? AND ?", condRange.From, condRange.To) + } else if condRange.From != 0.0 { + return query.Where("? <= JSON_EXTRACT(footprint, \"$."+condName+"\")", condRange.From) + } else if condRange.To != 0.0 { + return query.Where("JSON_EXTRACT(footprint, \"$."+condName+"\") <= ?", condRange.To) + } else { + return query + } } // buildStringCondition creates filters for string fields supporting equality, diff --git a/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql b/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql index bd465bcb..43afe40b 100644 --- a/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql +++ b/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql @@ -124,13 +124,15 @@ CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user); CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project); CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster); -- Cluster Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_cluster_starttime ON job (cluster, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_duration ON job (cluster, duration); CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes); CREATE INDEX IF NOT EXISTS jobs_cluster_numhwthreads ON job (cluster, num_hwthreads); CREATE INDEX IF NOT EXISTS jobs_cluster_numacc ON job (cluster, num_acc); CREATE INDEX IF NOT EXISTS jobs_cluster_energy ON job (cluster, energy); +-- Cluster Time Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_duration_starttime ON job (cluster, duration, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_starttime_duration ON job (cluster, start_time, duration); + -- Cluster+Partition Filter CREATE INDEX IF NOT EXISTS jobs_cluster_partition_user ON job (cluster, cluster_partition, hpc_user); CREATE INDEX IF NOT EXISTS jobs_cluster_partition_project ON job (cluster, cluster_partition, project); @@ -138,35 +140,41 @@ CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, clus CREATE INDEX IF NOT EXISTS jobs_cluster_partition_shared ON job (cluster, cluster_partition, shared); -- Cluster+Partition Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, cluster_partition, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, cluster_partition, duration); CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes); CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads); CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc); CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy); +-- Cluster+Partition Time Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration_starttime ON job (cluster, cluster_partition, duration, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime_duration ON job (cluster, cluster_partition, start_time, duration); + -- Cluster+JobState Filter CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project); -- Cluster+JobState Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration ON job (cluster, job_state, duration); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numhwthreads ON job (cluster, job_state, num_hwthreads); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_state, num_acc); CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy); +-- Cluster+JobState Time Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime_duration ON job (cluster, job_state, start_time, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration_starttime ON job (cluster, job_state, duration, start_time); + -- Cluster+Shared Filter CREATE INDEX IF NOT EXISTS jobs_cluster_shared_user ON job (cluster, shared, hpc_user); CREATE INDEX IF NOT EXISTS jobs_cluster_shared_project ON job (cluster, shared, project); -- Cluster+Shared Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_cluster_shared_starttime ON job (cluster, shared, start_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_shared_duration ON job (cluster, shared, duration); CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numnodes ON job (cluster, shared, num_nodes); CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numhwthreads ON job (cluster, shared, num_hwthreads); CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numacc ON job (cluster, shared, num_acc); CREATE INDEX IF NOT EXISTS jobs_cluster_shared_energy ON job (cluster, shared, energy); +-- Cluster+Shared Time Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_starttime_duration ON job (cluster, shared, start_time, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_duration_starttime ON job (cluster, shared, duration, start_time); + -- User Filter -- User Filter Sorting CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time); @@ -179,35 +187,41 @@ CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy); -- Project Filter CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user); -- Project Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time); -CREATE INDEX IF NOT EXISTS jobs_project_duration ON job (project, duration); CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes); CREATE INDEX IF NOT EXISTS jobs_project_numhwthreads ON job (project, num_hwthreads); CREATE INDEX IF NOT EXISTS jobs_project_numacc ON job (project, num_acc); CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy); +-- Cluster+Shared Time Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_project_starttime_duration ON job (project, start_time, duration); +CREATE INDEX IF NOT EXISTS jobs_project_duration_starttime ON job (project, duration, start_time); + -- JobState Filter CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user); CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project); -- JobState Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime ON job (job_state, start_time); -CREATE INDEX IF NOT EXISTS jobs_jobstate_duration ON job (job_state, duration); CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes); CREATE INDEX IF NOT EXISTS jobs_jobstate_numhwthreads ON job (job_state, num_hwthreads); CREATE INDEX IF NOT EXISTS jobs_jobstate_numacc ON job (job_state, num_acc); CREATE INDEX IF NOT EXISTS jobs_jobstate_energy ON job (job_state, energy); +-- Cluster+Shared Time Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime_duration ON job (job_state, start_time, duration); +CREATE INDEX IF NOT EXISTS jobs_jobstate_duration_starttime ON job (job_state, duration, start_time); + -- Shared Filter CREATE INDEX IF NOT EXISTS jobs_shared_user ON job (shared, hpc_user); CREATE INDEX IF NOT EXISTS jobs_shared_project ON job (shared, project); -- Shared Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_shared_starttime ON job (shared, start_time); -CREATE INDEX IF NOT EXISTS jobs_shared_duration ON job (shared, duration); CREATE INDEX IF NOT EXISTS jobs_shared_numnodes ON job (shared, num_nodes); CREATE INDEX IF NOT EXISTS jobs_shared_numhwthreads ON job (shared, num_hwthreads); CREATE INDEX IF NOT EXISTS jobs_shared_numacc ON job (shared, num_acc); CREATE INDEX IF NOT EXISTS jobs_shared_energy ON job (shared, energy); +-- Cluster+Shared Time Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_shared_starttime_duration ON job (shared, start_time, duration); +CREATE INDEX IF NOT EXISTS jobs_shared_duration_starttime ON job (shared, duration, start_time); + -- ArrayJob Filter CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time); CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time); @@ -226,6 +240,10 @@ CREATE INDEX IF NOT EXISTS jobs_numhwthreads_duration ON job (num_hwthreads, dur CREATE INDEX IF NOT EXISTS jobs_numacc_duration ON job (num_acc, duration); CREATE INDEX IF NOT EXISTS jobs_energy_duration ON job (energy, duration); +-- Backup Indices For High Variety Columns +CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time); +CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration); + -- Notes: -- Cluster+Partition+Jobstate Filter: Tested -> Full Array Of Combinations non-required -- Cluster+JobState+Shared Filter: Tested -> No further timing improvement diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 88c38eb1..b8f6de95 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -277,10 +277,22 @@ func buildFilterPresets(query url.Values) map[string]interface{} { if query.Get("duration") != "" { parts := strings.Split(query.Get("duration"), "-") if len(parts) == 2 { - a, e1 := strconv.Atoi(parts[0]) - b, e2 := strconv.Atoi(parts[1]) - if e1 == nil && e2 == nil { - filterPresets["duration"] = map[string]int{"from": a, "to": b} + if parts[0] == "lessthan" { + lt, lte := strconv.Atoi(parts[1]) + if lte == nil { + filterPresets["duration"] = map[string]int{"lessThan": lt, "from": 0, "to": 0} + } + } else if parts[0] == "morethan" { + mt, mte := strconv.Atoi(parts[1]) + if mte == nil { + filterPresets["duration"] = map[string]int{"moreThan": mt, "from": 0, "to": 0} + } + } else { + a, e1 := strconv.Atoi(parts[0]) + b, e2 := strconv.Atoi(parts[1]) + if e1 == nil && e2 == nil { + filterPresets["duration"] = map[string]int{"from": a, "to": b} + } } } } diff --git a/web/frontend/src/generic/Filters.svelte b/web/frontend/src/generic/Filters.svelte index adb865f3..c79a56e4 100644 --- a/web/frontend/src/generic/Filters.svelte +++ b/web/frontend/src/generic/Filters.svelte @@ -192,14 +192,14 @@ items.push({ startTime: { range: filters.startTime.range }, }); - if (filters.duration.from || filters.duration.to) + if (filters.duration.from && filters.duration.to) items.push({ duration: { from: filters.duration.from, to: filters.duration.to }, }); if (filters.duration.lessThan) - items.push({ duration: { from: 0, to: filters.duration.lessThan } }); + items.push({ duration: { to: filters.duration.lessThan, from: 0 } }); if (filters.duration.moreThan) - items.push({ duration: { from: filters.duration.moreThan, to: 604800 } }); // 7 days to include special jobs with long runtimes + items.push({ duration: { to: 0, from: filters.duration.moreThan } }); if (filters.energy.from || filters.energy.to) items.push({ energy: { from: filters.energy.from, to: filters.energy.to }, @@ -266,9 +266,9 @@ if (filters.duration.from && filters.duration.to) opts.push(`duration=${filters.duration.from}-${filters.duration.to}`); if (filters.duration.lessThan) - opts.push(`duration=0-${filters.duration.lessThan}`); + opts.push(`duration=lessthan-${filters.duration.lessThan}`); if (filters.duration.moreThan) - opts.push(`duration=${filters.duration.moreThan}-604800`); + opts.push(`duration=morethan-${filters.duration.moreThan}`); if (filters.tags.length != 0) for (let tag of filters.tags) opts.push(`tag=${tag}`); if (filters.numNodes.from && filters.numNodes.to) diff --git a/web/frontend/src/generic/filters/Duration.svelte b/web/frontend/src/generic/filters/Duration.svelte index 34f7afb2..78744de5 100644 --- a/web/frontend/src/generic/filters/Duration.svelte +++ b/web/frontend/src/generic/filters/Duration.svelte @@ -31,14 +31,16 @@ setFilter } = $props(); + /* States */ + let lessState = $state({ hours:0, mins:0 }); + let moreState = $state({ hours:0, mins:0 }); + let fromState = $state({ hours:0, mins:0 }); + let toState = $state({ hours:0, mins:0 }); + /* Derived */ let pendingDuration = $derived(presetDuration); - let lessState = $derived(secsToHoursAndMins(presetDuration?.lessThan)); - let moreState = $derived(secsToHoursAndMins(presetDuration?.moreThan)); - let fromState = $derived(secsToHoursAndMins(presetDuration?.from)); - let toState = $derived(secsToHoursAndMins(presetDuration?.to)); - const lessDisabled = $derived( + let lessDisabled = $derived( moreState.hours !== 0 || moreState.mins !== 0 || fromState.hours !== 0 || @@ -47,7 +49,7 @@ toState.mins !== 0 ); - const moreDisabled = $derived( + let moreDisabled = $derived( lessState.hours !== 0 || lessState.mins !== 0 || fromState.hours !== 0 || @@ -56,13 +58,27 @@ toState.mins !== 0 ); - const betweenDisabled = $derived( + let betweenDisabled = $derived( moreState.hours !== 0 || moreState.mins !== 0 || lessState.hours !== 0 || lessState.mins !== 0 ) + /* Effects */ + $effect(() => { + lessState = secsToHoursAndMins(pendingDuration?.lessThan); + }); + $effect(() => { + moreState = secsToHoursAndMins(pendingDuration?.moreThan); + }); + $effect(() => { + fromState = secsToHoursAndMins(pendingDuration?.from); + }); + $effect(() => { + toState = secsToHoursAndMins(pendingDuration?.to); + }); + /* Functions */ function resetPending() { pendingDuration = { From 4aa337ccc8836028087c341a51659a042d640b93 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 26 Jan 2026 16:05:42 +0100 Subject: [PATCH 2/3] fix missing index change --- .../repository/migrations/sqlite3/09_add-job-cache.up.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql b/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql index 43afe40b..6e1ac009 100644 --- a/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql +++ b/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql @@ -177,13 +177,15 @@ CREATE INDEX IF NOT EXISTS jobs_cluster_shared_duration_starttime ON job (cluste -- User Filter -- User Filter Sorting -CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time); -CREATE INDEX IF NOT EXISTS jobs_user_duration ON job (hpc_user, duration); CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes); CREATE INDEX IF NOT EXISTS jobs_user_numhwthreads ON job (hpc_user, num_hwthreads); CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (hpc_user, num_acc); CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy); +-- Cluster+Shared Time Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_user_starttime_duration ON job (hpc_user, start_time, duration); +CREATE INDEX IF NOT EXISTS jobs_user_duration_starttime ON job (hpc_user, duration, start_time); + -- Project Filter CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user); -- Project Filter Sorting From 934bc13c2cbd75ebc1bb91427dcfd845f3c5212f Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 26 Jan 2026 18:09:54 +0100 Subject: [PATCH 3/3] add fallback case to extendedLegend render --- web/frontend/src/systems/nodelist/NodeListRow.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/frontend/src/systems/nodelist/NodeListRow.svelte b/web/frontend/src/systems/nodelist/NodeListRow.svelte index 0b34fbad..0e2aeb18 100644 --- a/web/frontend/src/systems/nodelist/NodeListRow.svelte +++ b/web/frontend/src/systems/nodelist/NodeListRow.svelte @@ -116,7 +116,7 @@ pendingExtendedLegendData = {}; for (const accId of accSet) { - const matchJob = $nodeJobsData.data.jobs.items.find((i) => i.resources.find((r) => r.accelerators.includes(accId))) + const matchJob = $nodeJobsData?.data?.jobs?.items?.find((i) => i?.resources?.find((r) => r?.accelerators?.includes(accId))) || null const matchUser = matchJob?.user ? matchJob.user : null pendingExtendedLegendData[accId] = { user: (scrambleNames && matchUser)