diff --git a/internal/repository/migration.go b/internal/repository/migration.go index 0f99889e..23b9939c 100644 --- a/internal/repository/migration.go +++ b/internal/repository/migration.go @@ -21,10 +21,11 @@ import ( // is added to internal/repository/migrations/sqlite3/. // // Version history: -// - Version 10: Current version +// - Version 11: Optimize job table indexes (reduce from ~78 to 48) +// - Version 10: Node table // // Migration files are embedded at build time from the migrations directory. -const Version uint = 10 +const Version uint = 11 //go:embed migrations/* var migrationFiles embed.FS diff --git a/internal/repository/migrations/sqlite3/11_optimize-indexes.down.sql b/internal/repository/migrations/sqlite3/11_optimize-indexes.down.sql new file mode 100644 index 00000000..770b3c1a --- /dev/null +++ b/internal/repository/migrations/sqlite3/11_optimize-indexes.down.sql @@ -0,0 +1,56 @@ +-- Migration 11 DOWN: Restore indexes from migration 09 + +-- ============================================================ +-- Recreate all removed indexes from migration 09 +-- ============================================================ + +-- Cluster+Partition Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy); + +-- Cluster+JobState Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numhwthreads ON job (cluster, job_state, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_state, num_acc); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy); + +-- Cluster+Shared Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numnodes ON job (cluster, shared, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numhwthreads ON job (cluster, shared, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_numacc ON job (cluster, shared, num_acc); +CREATE INDEX IF NOT EXISTS jobs_cluster_shared_energy ON job (cluster, shared, energy); + +-- User Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_user_numhwthreads ON job (hpc_user, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (hpc_user, num_acc); +CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy); + +-- Project Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_project_numhwthreads ON job (project, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_project_numacc ON job (project, num_acc); +CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy); + +-- JobState Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_jobstate_numhwthreads ON job (job_state, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_jobstate_numacc ON job (job_state, num_acc); +CREATE INDEX IF NOT EXISTS jobs_jobstate_energy ON job (job_state, energy); + +-- Shared Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_shared_numnodes ON job (shared, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_shared_numhwthreads ON job (shared, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_shared_numacc ON job (shared, num_acc); +CREATE INDEX IF NOT EXISTS jobs_shared_energy ON job (shared, energy); + +-- ArrayJob Filter +CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time); + +-- Backup Indices For High Variety Columns +CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration); + +-- Optimize DB index usage +PRAGMA optimize; diff --git a/internal/repository/migrations/sqlite3/11_optimize-indexes.up.sql b/internal/repository/migrations/sqlite3/11_optimize-indexes.up.sql new file mode 100644 index 00000000..df838da6 --- /dev/null +++ b/internal/repository/migrations/sqlite3/11_optimize-indexes.up.sql @@ -0,0 +1,61 @@ +-- Migration 11: Remove overly specific table indexes formerly used in sorting +-- When one or two indexed columns are used, sorting usually is fast +-- Reduces from ~78 indexes to 48 for better write performance, +-- reduced disk usage, and more reliable query planner decisions. +-- Requires ANALYZE to be run after migration (done automatically on startup). + +-- ============================================================ +-- Drop SELECTED existing job indexes (from migrations 08/09) +-- sqlite_autoindex_job_1 (UNIQUE constraint) is kept automatically +-- ============================================================ + +-- Cluster+Partition Filter Sorting +DROP INDEX IF EXISTS jobs_cluster_partition_numnodes; +DROP INDEX IF EXISTS jobs_cluster_partition_numhwthreads; +DROP INDEX IF EXISTS jobs_cluster_partition_numacc; +DROP INDEX IF EXISTS jobs_cluster_partition_energy; + +-- Cluster+JobState Filter Sorting +DROP INDEX IF EXISTS jobs_cluster_jobstate_numnodes; +DROP INDEX IF EXISTS jobs_cluster_jobstate_numhwthreads; +DROP INDEX IF EXISTS jobs_cluster_jobstate_numacc; +DROP INDEX IF EXISTS jobs_cluster_jobstate_energy; + +-- Cluster+Shared Filter Sorting +DROP INDEX IF EXISTS jobs_cluster_shared_numnodes; +DROP INDEX IF EXISTS jobs_cluster_shared_numhwthreads; +DROP INDEX IF EXISTS jobs_cluster_shared_numacc; +DROP INDEX IF EXISTS jobs_cluster_shared_energy; + +-- User Filter Sorting +DROP INDEX IF EXISTS jobs_user_numnodes; +DROP INDEX IF EXISTS jobs_user_numhwthreads; +DROP INDEX IF EXISTS jobs_user_numacc; +DROP INDEX IF EXISTS jobs_user_energy; + +-- Project Filter Sorting +DROP INDEX IF EXISTS jobs_project_numnodes; +DROP INDEX IF EXISTS jobs_project_numhwthreads; +DROP INDEX IF EXISTS jobs_project_numacc; +DROP INDEX IF EXISTS jobs_project_energy; + +-- JobState Filter Sorting +DROP INDEX IF EXISTS jobs_jobstate_numnodes; +DROP INDEX IF EXISTS jobs_jobstate_numhwthreads; +DROP INDEX IF EXISTS jobs_jobstate_numacc; +DROP INDEX IF EXISTS jobs_jobstate_energy; + +-- Shared Filter Sorting +DROP INDEX IF EXISTS jobs_shared_numnodes; +DROP INDEX IF EXISTS jobs_shared_numhwthreads; +DROP INDEX IF EXISTS jobs_shared_numacc; +DROP INDEX IF EXISTS jobs_shared_energy; + +-- ArrayJob Filter +DROP INDEX IF EXISTS jobs_cluster_arrayjobid_starttime; + +-- Backup Indices For High Variety Columns +DROP INDEX IF EXISTS jobs_duration; + +-- Optimize DB index usage +PRAGMA optimize;