diff --git a/internal/repository/jobQuery.go b/internal/repository/jobQuery.go index aacdebce..6cdb2860 100644 --- a/internal/repository/jobQuery.go +++ b/internal/repository/jobQuery.go @@ -63,12 +63,11 @@ func (r *JobRepository) QueryJobs( } } else { // Order by footprint JSON field values - query = query.Where("JSON_VALID(meta_data)") switch order.Order { case model.SortDirectionEnumAsc: - query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") ASC", field)) + query = query.OrderBy(fmt.Sprintf("json_extract(footprint, '$.%s') ASC", field)) case model.SortDirectionEnumDesc: - query = query.OrderBy(fmt.Sprintf("JSON_EXTRACT(footprint, \"$.%s\") DESC", field)) + query = query.OrderBy(fmt.Sprintf("json_extract(footprint, '$.%s') DESC", field)) default: return nil, errors.New("invalid sorting order for footprint") } @@ -336,13 +335,12 @@ func buildTimeCondition(field string, cond *config.TimeRange, query sq.SelectBui // buildFloatJSONCondition creates a filter on a numeric field within the footprint JSON column, using BETWEEN only if required. func buildFloatJSONCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder { - query = query.Where("JSON_VALID(footprint)") if cond.From != 1.0 && cond.To != 0.0 { - return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") BETWEEN ? AND ?", cond.From, cond.To) + return query.Where("json_extract(footprint, '$."+field+"') BETWEEN ? AND ?", cond.From, cond.To) } else if cond.From != 1.0 && cond.To == 0.0 { - return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") >= ?", cond.From) + return query.Where("json_extract(footprint, '$."+field+"') >= ?", cond.From) } else if cond.From == 1.0 && cond.To != 0.0 { - return query.Where("JSON_EXTRACT(footprint, \"$."+field+"\") <= ?", cond.To) + return query.Where("json_extract(footprint, '$."+field+"') <= ?", cond.To) } else { return query } diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 0f99889e..b9738a0d 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -21,10 +21,11 @@ import ( // is added to internal/repository/migrations/sqlite3/. // // Version history: -// - Version 10: Current version +// - Version 11: Add expression indexes on footprint JSON fields +// - Version 10: Previous version // // Migration files are embedded at build time from the migrations directory. -const Version uint = 10 +const Version uint = 11 //go:embed migrations/* var migrationFiles embed.FS diff --git a/internal/repository/migrations/sqlite3/11_add-footprint-indexes.down.sql b/internal/repository/migrations/sqlite3/11_add-footprint-indexes.down.sql new file mode 100644 index 00000000..f3f4dc06 --- /dev/null +++ b/internal/repository/migrations/sqlite3/11_add-footprint-indexes.down.sql @@ -0,0 +1,15 @@ +-- Drop standalone expression indexes +DROP INDEX IF EXISTS jobs_fp_flops_any_avg; +DROP INDEX IF EXISTS jobs_fp_mem_bw_avg; +DROP INDEX IF EXISTS jobs_fp_mem_used_max; +DROP INDEX IF EXISTS jobs_fp_cpu_load_avg; +DROP INDEX IF EXISTS jobs_fp_net_bw_avg; +DROP INDEX IF EXISTS jobs_fp_net_data_vol_total; +DROP INDEX IF EXISTS jobs_fp_file_bw_avg; +DROP INDEX IF EXISTS jobs_fp_file_data_vol_total; + +-- Drop composite indexes +DROP INDEX IF EXISTS jobs_cluster_fp_cpu_load_avg; +DROP INDEX IF EXISTS jobs_cluster_fp_flops_any_avg; +DROP INDEX IF EXISTS jobs_cluster_fp_mem_bw_avg; +DROP INDEX IF EXISTS jobs_cluster_fp_mem_used_max; diff --git a/internal/repository/migrations/sqlite3/11_add-footprint-indexes.up.sql b/internal/repository/migrations/sqlite3/11_add-footprint-indexes.up.sql new file mode 100644 index 00000000..12e064ee --- /dev/null +++ b/internal/repository/migrations/sqlite3/11_add-footprint-indexes.up.sql @@ -0,0 +1,19 @@ +-- Expression indexes on footprint JSON fields for WHERE and ORDER BY optimization. +-- SQLite matches expressions textually, so queries must use exactly: +-- json_extract(footprint, '$.field') + +-- Standalone expression indexes (for filtering and sorting) +CREATE INDEX IF NOT EXISTS jobs_fp_flops_any_avg ON job (json_extract(footprint, '$.flops_any_avg')); +CREATE INDEX IF NOT EXISTS jobs_fp_mem_bw_avg ON job (json_extract(footprint, '$.mem_bw_avg')); +CREATE INDEX IF NOT EXISTS jobs_fp_mem_used_max ON job (json_extract(footprint, '$.mem_used_max')); +CREATE INDEX IF NOT EXISTS jobs_fp_cpu_load_avg ON job (json_extract(footprint, '$.cpu_load_avg')); +CREATE INDEX IF NOT EXISTS jobs_fp_net_bw_avg ON job (json_extract(footprint, '$.net_bw_avg')); +CREATE INDEX IF NOT EXISTS jobs_fp_net_data_vol_total ON job (json_extract(footprint, '$.net_data_vol_total')); +CREATE INDEX IF NOT EXISTS jobs_fp_file_bw_avg ON job (json_extract(footprint, '$.file_bw_avg')); +CREATE INDEX IF NOT EXISTS jobs_fp_file_data_vol_total ON job (json_extract(footprint, '$.file_data_vol_total')); + +-- Composite indexes with cluster (for common filter+sort combinations) +CREATE INDEX IF NOT EXISTS jobs_cluster_fp_cpu_load_avg ON job (cluster, json_extract(footprint, '$.cpu_load_avg')); +CREATE INDEX IF NOT EXISTS jobs_cluster_fp_flops_any_avg ON job (cluster, json_extract(footprint, '$.flops_any_avg')); +CREATE INDEX IF NOT EXISTS jobs_cluster_fp_mem_bw_avg ON job (cluster, json_extract(footprint, '$.mem_bw_avg')); +CREATE INDEX IF NOT EXISTS jobs_cluster_fp_mem_used_max ON job (cluster, json_extract(footprint, '$.mem_used_max')); diff --git a/internal/repository/stats.go b/internal/repository/stats.go index 0ceb92f2..b792c1f6 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -921,16 +921,14 @@ func (r *JobRepository) jobsMetricStatisticsHistogram( // Special case: value == peak would create bin N+1, so we test for equality // and multiply peak by 0.999999999 to force it into bin N. binQuery := fmt.Sprintf(`CAST( - ((case when json_extract(footprint, "$.%s") = %f then %f*0.999999999 else json_extract(footprint, "$.%s") end) / %f) + ((case when json_extract(footprint, '$.%s') = %f then %f*0.999999999 else json_extract(footprint, '$.%s') end) / %f) * %v as INTEGER )`, (metric + "_" + footprintStat), peak, peak, (metric + "_" + footprintStat), peak, *bins) mainQuery := sq.Select( fmt.Sprintf(`%s + 1 as bin`, binQuery), `count(*) as count`, - ).From("job").Where( - "JSON_VALID(footprint)", - ).Where(fmt.Sprintf(`json_extract(footprint, "$.%s") is not null and json_extract(footprint, "$.%s") <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak)) + ).From("job").Where(fmt.Sprintf(`json_extract(footprint, '$.%s') is not null and json_extract(footprint, '$.%s') <= %f`, (metric + "_" + footprintStat), (metric + "_" + footprintStat), peak)) mainQuery, qerr := SecurityCheck(ctx, mainQuery) if qerr != nil {