Consolidate migrations

Entire-Checkpoint: a3dba4105838

internal/repository/migration.go

@@ -21,14 +21,11 @@ import (
 // is added to internal/repository/migrations/sqlite3/.
 //
 // Version history:
-//   - Version 14: Partial covering indexes for running jobs (tiny B-tree vs millions)
-//   - Version 13: Add covering indexes for status/dashboard queries (cluster, job_state, ...)
-//   - Version 12: Add covering index for stats queries (cluster, start_time, hpc_user, ...)
-//   - Version 11: Optimize job table indexes (reduce from ~78 to 48)
+//   - Version 11: Optimize job table indexes (reduce from ~78 to 48, add covering/partial indexes)
 //   - Version 10: Node table
 //
 // Migration files are embedded at build time from the migrations directory.
-const Version uint = 14
+const Version uint = 11
 
 //go:embed migrations/*
 var migrationFiles embed.FS

internal/repository/migrations/sqlite3/11_optimize-indexes.down.sql

@@ -1,8 +1,26 @@
 -- Migration 11 DOWN: Restore indexes from migration 09
 
--- ============================================================
+-- Drop partial indexes for running jobs
+DROP INDEX IF EXISTS jobs_running_user_stats;
+DROP INDEX IF EXISTS jobs_running_project_stats;
+DROP INDEX IF EXISTS jobs_running_subcluster_stats;
+
+-- Drop covering status indexes, restore 3-col indexes
+DROP INDEX IF EXISTS jobs_cluster_jobstate_user_stats;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_project_stats;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_subcluster_stats;
+
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user
+  ON job (cluster, job_state, hpc_user);
+
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project
+  ON job (cluster, job_state, project);
+
+-- Drop covering stats indexes
+DROP INDEX IF EXISTS jobs_cluster_user_starttime_stats;
+DROP INDEX IF EXISTS jobs_cluster_project_starttime_stats;
+
 -- Recreate all removed indexes from migration 09
--- ============================================================
 
 -- Cluster+Partition Filter Sorting
 CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes);
@@ -52,5 +70,4 @@ CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, ar
 -- Backup Indices For High Variety Columns
 CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration);
 
--- Optimize DB index usage
 PRAGMA optimize;

internal/repository/migrations/sqlite3/11_optimize-indexes.up.sql

@@ -1,8 +1,8 @@
--- Migration 11: Remove overly specific table indexes formerly used in sorting
--- When one or two indexed columns are used, sorting usually is fast
--- Reduces from ~78 indexes to 48 for better write performance,
--- reduced disk usage, and more reliable query planner decisions.
--- Requires ANALYZE to be run after migration (done automatically on startup).
+-- Migration 11: Optimize job table indexes
+--   - Remove overly specific sorting indexes (reduces ~78 → 48)
+--   - Add covering index for grouped stats queries
+--   - Add covering indexes for status/dashboard queries
+--   - Add partial covering indexes for running jobs (tiny B-tree)
 
 -- ============================================================
 -- Drop SELECTED existing job indexes (from migrations 08/09)
@@ -57,5 +57,54 @@ DROP INDEX IF EXISTS jobs_cluster_arrayjobid_starttime;
 -- Backup Indices For High Variety Columns
 DROP INDEX IF EXISTS jobs_duration;
 
--- Optimize DB index usage
+-- ============================================================
+-- Covering indexes for grouped stats queries
+-- Column order: cluster (equality), hpc_user/project (GROUP BY), start_time (range scan)
+-- Includes aggregated columns to avoid main table lookups entirely.
+-- ============================================================
+
+CREATE INDEX IF NOT EXISTS jobs_cluster_user_starttime_stats
+  ON job (cluster, hpc_user, start_time, duration, job_state, num_nodes, num_hwthreads, num_acc);
+
+CREATE INDEX IF NOT EXISTS jobs_cluster_project_starttime_stats
+  ON job (cluster, project, start_time, duration, job_state, num_nodes, num_hwthreads, num_acc);
+
+-- ============================================================
+-- Covering indexes for status/dashboard queries
+-- Column order: cluster (equality), job_state (equality), grouping col, then aggregated columns
+-- These indexes allow the status views to be served entirely from index scans.
+-- ============================================================
+
+-- Drop 3-col indexes that are subsumed by the covering indexes below
+DROP INDEX IF EXISTS jobs_cluster_jobstate_user;
+DROP INDEX IF EXISTS jobs_cluster_jobstate_project;
+
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user_stats
+  ON job (cluster, job_state, hpc_user, duration, start_time, num_nodes, num_hwthreads, num_acc);
+
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project_stats
+  ON job (cluster, job_state, project, duration, start_time, num_nodes, num_hwthreads, num_acc);
+
+CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_subcluster_stats
+  ON job (cluster, job_state, subcluster, duration, start_time, num_nodes, num_hwthreads, num_acc);
+
+-- ============================================================
+-- Partial covering indexes for running jobs
+-- Only running jobs are in the B-tree, so these indexes are tiny compared to
+-- the full-table indexes above. SQLite uses them when the query contains the
+-- literal `job_state = 'running'` (not a parameter placeholder).
+-- ============================================================
+
+CREATE INDEX IF NOT EXISTS jobs_running_user_stats
+  ON job (cluster, hpc_user, num_nodes, num_hwthreads, num_acc, duration, start_time)
+  WHERE job_state = 'running';
+
+CREATE INDEX IF NOT EXISTS jobs_running_project_stats
+  ON job (cluster, project, num_nodes, num_hwthreads, num_acc, duration, start_time)
+  WHERE job_state = 'running';
+
+CREATE INDEX IF NOT EXISTS jobs_running_subcluster_stats
+  ON job (cluster, subcluster, num_nodes, num_hwthreads, num_acc, duration, start_time)
+  WHERE job_state = 'running';
+
 PRAGMA optimize;

internal/repository/migrations/sqlite3/12_stats-covering-index.down.sql

@@ -1,4 +0,0 @@
-DROP INDEX IF EXISTS jobs_cluster_user_starttime_stats;
-DROP INDEX IF EXISTS jobs_cluster_project_starttime_stats;
-
-PRAGMA optimize;

internal/repository/migrations/sqlite3/12_stats-covering-index.up.sql

@@ -1,11 +0,0 @@
--- Migration 12: Add covering index for grouped stats queries
--- Column order: cluster (equality), hpc_user (GROUP BY), start_time (range scan)
--- Includes aggregated columns to avoid main table lookups entirely.
-
-CREATE INDEX IF NOT EXISTS jobs_cluster_user_starttime_stats
-  ON job (cluster, hpc_user, start_time, duration, job_state, num_nodes, num_hwthreads, num_acc);
-
-CREATE INDEX IF NOT EXISTS jobs_cluster_project_starttime_stats
-  ON job (cluster, project, start_time, duration, job_state, num_nodes, num_hwthreads, num_acc);
-
-PRAGMA optimize;

internal/repository/migrations/sqlite3/13_status-covering-indexes.down.sql

@@ -1,14 +0,0 @@
--- Reverse migration 13: Remove covering status indexes, restore 3-col indexes
-
-DROP INDEX IF EXISTS jobs_cluster_jobstate_user_stats;
-DROP INDEX IF EXISTS jobs_cluster_jobstate_project_stats;
-DROP INDEX IF EXISTS jobs_cluster_jobstate_subcluster_stats;
-
--- Restore the original 3-col indexes
-CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user
-  ON job (cluster, job_state, hpc_user);
-
-CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project
-  ON job (cluster, job_state, project);
-
-  PRAGMA optimize;

internal/repository/migrations/sqlite3/13_status-covering-indexes.up.sql

@@ -1,18 +0,0 @@
--- Migration 13: Add covering indexes for status/dashboard queries
--- Column order: cluster (equality), job_state (equality), grouping col, then aggregated columns
--- These indexes allow the status views to be served entirely from index scans.
-
-CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user_stats
-  ON job (cluster, job_state, hpc_user, duration, start_time, num_nodes, num_hwthreads, num_acc);
-
-CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project_stats
-  ON job (cluster, job_state, project, duration, start_time, num_nodes, num_hwthreads, num_acc);
-
-CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_subcluster_stats
-  ON job (cluster, job_state, subcluster, duration, start_time, num_nodes, num_hwthreads, num_acc);
-
--- Drop 3-col indexes that are now subsumed by the covering indexes above
-DROP INDEX IF EXISTS jobs_cluster_jobstate_user;
-DROP INDEX IF EXISTS jobs_cluster_jobstate_project;
-
-PRAGMA optimize;

internal/repository/migrations/sqlite3/14_running-partial-indexes.down.sql

@@ -1,7 +0,0 @@
--- Reverse migration 14: Drop partial indexes for running jobs
-
-DROP INDEX IF EXISTS jobs_running_user_stats;
-DROP INDEX IF EXISTS jobs_running_project_stats;
-DROP INDEX IF EXISTS jobs_running_subcluster_stats;
-
-PRAGMA optimize;

internal/repository/migrations/sqlite3/14_running-partial-indexes.up.sql

@@ -1,18 +0,0 @@
--- Migration 14: Partial covering indexes for running jobs
--- Only running jobs are in the B-tree, so these indexes are tiny compared to
--- the full-table indexes from migration 13. SQLite uses them when the query
--- contains the literal `job_state = 'running'` (not a parameter placeholder).
-
-CREATE INDEX IF NOT EXISTS jobs_running_user_stats
-  ON job (cluster, hpc_user, num_nodes, num_hwthreads, num_acc, duration, start_time)
-  WHERE job_state = 'running';
-
-CREATE INDEX IF NOT EXISTS jobs_running_project_stats
-  ON job (cluster, project, num_nodes, num_hwthreads, num_acc, duration, start_time)
-  WHERE job_state = 'running';
-
-CREATE INDEX IF NOT EXISTS jobs_running_subcluster_stats
-  ON job (cluster, subcluster, num_nodes, num_hwthreads, num_acc, duration, start_time)
-  WHERE job_state = 'running';
-
-PRAGMA optimize;