ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2026-03-10 18:17:30 +01:00
Code Issues Packages Releases Activity

Optimize queries on footprint attributes

Browse Source 
Entire-Checkpoint: 81bf5e62e311
This commit is contained in:
Jan Eitzinger
2026-03-10 05:50:21 +01:00
parent d2bc046fc6
commit 907fccb3e3
5 changed files with 44 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
internal/repository/jobQuery.go
View File

@@ -63,12 +63,11 @@ func (r *JobRepository) QueryJobs(
}
} else {
// Order by footprint JSON field values
query = query.Where("JSON_VALID(meta_data)")
switch order.Order {
case model.SortDirectionEnumAsc:
query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") ASC", field))
query = query.OrderBy(fmt.Sprintf("json_extract(footprint, '$.%s') ASC", field))
case model.SortDirectionEnumDesc:
query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") DESC", field))
query = query.OrderBy(fmt.Sprintf("json_extract(footprint, '$.%s') DESC", field))
default:
return nil, errors.New("invalid sorting order for footprint")
}
@@ -336,13 +335,12 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui
// buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column, using BETWEEN only if required.
func buildFloatJSONCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
query = query.Where("JSON_VALID(footprint)")
if cond.From != 1.0 && cond.To != 0.0 {
return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") BETWEEN ? AND ?", cond.From, cond.To)
return query.Where("json_extract(footprint, '$."+field+"') BETWEEN ? AND ?", cond.From, cond.To)
} else if cond.From != 1.0 && cond.To == 0.0 {
return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") >= ?", cond.From)
return query.Where("json_extract(footprint, '$."+field+"') >= ?", cond.From)
} else if cond.From == 1.0 && cond.To != 0.0 {
return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") <= ?", cond.To)
return query.Where("json_extract(footprint, '$."+field+"') <= ?", cond.To)
} else {
return query
}

5
internal/repository/migration.go
View File

@@ -21,10 +21,11 @@ import (
// is added to internal/repository/migrations/sqlite3/.
//
// Version history:
// - Version 10: Current version
// - Version 11: Add expression indexes on footprint JSON fields
// - Version 10: Previous version
//
// Migration files are embedded at build time from the migrations directory.
const Version uint = 10
const Version uint = 11
//go:embed migrations/*
var migrationFiles embed.FS

15
internal/repository/migrations/sqlite3/11_add-footprint-indexes.down.sql Normal file
View File

@@ -0,0 +1,15 @@
-- Drop standalone expression indexes
DROP INDEX IF EXISTS jobs_fp_flops_any_avg;
DROP INDEX IF EXISTS jobs_fp_mem_bw_avg;
DROP INDEX IF EXISTS jobs_fp_mem_used_max;
DROP INDEX IF EXISTS jobs_fp_cpu_load_avg;
DROP INDEX IF EXISTS jobs_fp_net_bw_avg;
DROP INDEX IF EXISTS jobs_fp_net_data_vol_total;
DROP INDEX IF EXISTS jobs_fp_file_bw_avg;
DROP INDEX IF EXISTS jobs_fp_file_data_vol_total;
-- Drop composite indexes
DROP INDEX IF EXISTS jobs_cluster_fp_cpu_load_avg;
DROP INDEX IF EXISTS jobs_cluster_fp_flops_any_avg;
DROP INDEX IF EXISTS jobs_cluster_fp_mem_bw_avg;
DROP INDEX IF EXISTS jobs_cluster_fp_mem_used_max;

19
internal/repository/migrations/sqlite3/11_add-footprint-indexes.up.sql Normal file
View File

@@ -0,0 +1,19 @@
-- Expression indexes on footprint JSON fields for WHERE and ORDER BY optimization.
-- SQLite matches expressions textually, so queries must use exactly:
-- json_extract(footprint, '$.field')
-- Standalone expression indexes (for filtering and sorting)
CREATE INDEX IF NOT EXISTS jobs_fp_flops_any_avg ON job (json_extract(footprint, '$.flops_any_avg'));
CREATE INDEX IF NOT EXISTS jobs_fp_mem_bw_avg ON job (json_extract(footprint, '$.mem_bw_avg'));
CREATE INDEX IF NOT EXISTS jobs_fp_mem_used_max ON job (json_extract(footprint, '$.mem_used_max'));
CREATE INDEX IF NOT EXISTS jobs_fp_cpu_load_avg ON job (json_extract(footprint, '$.cpu_load_avg'));
CREATE INDEX IF NOT EXISTS jobs_fp_net_bw_avg ON job (json_extract(footprint, '$.net_bw_avg'));
CREATE INDEX IF NOT EXISTS jobs_fp_net_data_vol_total ON job (json_extract(footprint, '$.net_data_vol_total'));
CREATE INDEX IF NOT EXISTS jobs_fp_file_bw_avg ON job (json_extract(footprint, '$.file_bw_avg'));
CREATE INDEX IF NOT EXISTS jobs_fp_file_data_vol_total ON job (json_extract(footprint, '$.file_data_vol_total'));
-- Composite indexes with cluster (for common filter+sort combinations)
CREATE INDEX IF NOT EXISTS jobs_cluster_fp_cpu_load_avg ON job (cluster, json_extract(footprint, '$.cpu_load_avg'));
CREATE INDEX IF NOT EXISTS jobs_cluster_fp_flops_any_avg ON job (cluster, json_extract(footprint, '$.flops_any_avg'));
CREATE INDEX IF NOT EXISTS jobs_cluster_fp_mem_bw_avg ON job (cluster, json_extract(footprint, '$.mem_bw_avg'));
CREATE INDEX IF NOT EXISTS jobs_cluster_fp_mem_used_max ON job (cluster, json_extract(footprint, '$.mem_used_max'));

6
internal/repository/stats.go
View File

@@ -921,16 +921,14 @@ func (r *JobRepository) jobsMetricStatisticsHistogram(
// Special case: value == peak would create bin N+1, so we test for equality
// and multiply peak by 0.999999999 to force it into bin N.
binQuery := fmt.Sprintf(`CAST(
((case when json_extract(footprint, "$.%s") = %f then %f*0.999999999 else json_extract(footprint, "$.%s") end) / %f)
((case when json_extract(footprint, '$.%s') = %f then %f*0.999999999 else json_extract(footprint, '$.%s') end) / %f)
* %v as INTEGER )`,
(metric + "_" + footprintStat), peak, peak, (metric + "_" + footprintStat), peak, *bins)
mainQuery := sq.Select(
fmt.Sprintf(`%s + 1 as bin`, binQuery),
`count(*) as count`,
).From("job").Where(
"JSON_VALID(footprint)",
).Where(fmt.Sprintf(`json_extract(footprint, "$.%s") is not null and json_extract(footprint, "$.%s") <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak))
).From("job").Where(fmt.Sprintf(`json_extract(footprint, '$.%s') is not null and json_extract(footprint, '$.%s') <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak))
mainQuery, qerr := SecurityCheck(ctx, mainQuery)
if qerr != nil {