From c4cf7e9707f74c4fc7d80a482c354c5c4e8457b5 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 21 Nov 2025 13:44:05 +0100 Subject: [PATCH] Recreate job table indices after copy, add node table timstamp indices --- .../sqlite3/09_add-job-cache.up.sql | 110 +++++++++++++++++- .../migrations/sqlite3/10_node-table.up.sql | 14 +-- 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql b/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql index ea55df7..d716c40 100644 --- a/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql +++ b/internal/repository/migrations/sqlite3/09_add-job-cache.up.sql @@ -115,5 +115,113 @@ INSERT INTO job_new ( FROM job; DROP TABLE lookup_exclusive; -DROP TABLE job; +DROP TABLE job; -- Deletes All Existing 'job' Indices; Recreate after Renaming ALTER TABLE job_new RENAME TO job; + +-- Recreate Indices from 08_add-footprint, include new submit_time indices +-- Cluster Filter +CREATE INDEX IF NOT EXISTS jobs_cluster ON job (cluster); +CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project); +CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster); +-- Cluster Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_starttime ON job (cluster, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_submittime ON job (cluster, submit_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_duration ON job (cluster, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_numnodes ON job (cluster, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_cluster_numhwthreads ON job (cluster, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_cluster_numacc ON job (cluster, num_acc); +CREATE INDEX IF NOT EXISTS jobs_cluster_energy ON job (cluster, energy); + +-- Cluster+Partition Filter +CREATE INDEX IF NOT EXISTS jobs_cluster_partition ON job (cluster, cluster_partition); +-- Cluster+Partition Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, cluster_partition, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_submittime ON job (cluster, cluster_partition, submit_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_duration ON job (cluster, cluster_partition, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numnodes ON job (cluster, cluster_partition, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numhwthreads ON job (cluster, cluster_partition, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_numacc ON job (cluster, cluster_partition, num_acc); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_energy ON job (cluster, cluster_partition, energy); + +-- Cluster+Partition+Jobstate Filter +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, cluster_partition, job_state); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_user ON job (cluster, cluster_partition, job_state, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_project ON job (cluster, cluster_partition, job_state, project); +-- Cluster+Partition+Jobstate Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_starttime ON job (cluster, cluster_partition, job_state, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_submittime ON job (cluster, cluster_partition, job_state, submit_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_duration ON job (cluster, cluster_partition, job_state, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numnodes ON job (cluster, cluster_partition, job_state, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numhwthreads ON job (cluster, cluster_partition, job_state, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_numacc ON job (cluster, cluster_partition, job_state, num_acc); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_energy ON job (cluster, cluster_partition, job_state, energy); + +-- Cluster+JobState Filter +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate ON job (cluster, job_state); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project); +-- Cluster+JobState Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_submittime ON job (cluster, job_state, submit_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_duration ON job (cluster, job_state, duration); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numnodes ON job (cluster, job_state, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numhwthreads ON job (cluster, job_state, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_numacc ON job (cluster, job_state, num_acc); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_energy ON job (cluster, job_state, energy); + +-- User Filter +CREATE INDEX IF NOT EXISTS jobs_user ON job (hpc_user); +-- User Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (hpc_user, start_time); +CREATE INDEX IF NOT EXISTS jobs_user_duration ON job (hpc_user, duration); +CREATE INDEX IF NOT EXISTS jobs_user_numnodes ON job (hpc_user, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_user_numhwthreads ON job (hpc_user, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_user_numacc ON job (hpc_user, num_acc); +CREATE INDEX IF NOT EXISTS jobs_user_energy ON job (hpc_user, energy); + +-- Project Filter +CREATE INDEX IF NOT EXISTS jobs_project ON job (project); +CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, hpc_user); +-- Project Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time); +CREATE INDEX IF NOT EXISTS jobs_project_duration ON job (project, duration); +CREATE INDEX IF NOT EXISTS jobs_project_numnodes ON job (project, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_project_numhwthreads ON job (project, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_project_numacc ON job (project, num_acc); +CREATE INDEX IF NOT EXISTS jobs_project_energy ON job (project, energy); + +-- JobState Filter +CREATE INDEX IF NOT EXISTS jobs_jobstate ON job (job_state); +CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, hpc_user); +CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project); +CREATE INDEX IF NOT EXISTS jobs_jobstate_cluster ON job (job_state, cluster); +-- JobState Filter Sorting +CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime ON job (job_state, start_time); +CREATE INDEX IF NOT EXISTS jobs_jobstate_duration ON job (job_state, duration); +CREATE INDEX IF NOT EXISTS jobs_jobstate_numnodes ON job (job_state, num_nodes); +CREATE INDEX IF NOT EXISTS jobs_jobstate_numhwthreads ON job (job_state, num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_jobstate_numacc ON job (job_state, num_acc); +CREATE INDEX IF NOT EXISTS jobs_jobstate_energy ON job (job_state, energy); + +-- ArrayJob Filter +CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time); + +-- Sorting without active filters +CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time); +CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration); +CREATE INDEX IF NOT EXISTS jobs_numnodes ON job (num_nodes); +CREATE INDEX IF NOT EXISTS jobs_numhwthreads ON job (num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_numacc ON job (num_acc); +CREATE INDEX IF NOT EXISTS jobs_energy ON job (energy); + +-- Single filters with default starttime sorting +CREATE INDEX IF NOT EXISTS jobs_duration_starttime ON job (duration, start_time); +CREATE INDEX IF NOT EXISTS jobs_numnodes_starttime ON job (num_nodes, start_time); +CREATE INDEX IF NOT EXISTS jobs_numhwthreads_starttime ON job (num_hwthreads, start_time); +CREATE INDEX IF NOT EXISTS jobs_numacc_starttime ON job (num_acc, start_time); +CREATE INDEX IF NOT EXISTS jobs_energy_starttime ON job (energy, start_time); + +-- Optimize DB index usage +PRAGMA optimize; \ No newline at end of file diff --git a/internal/repository/migrations/sqlite3/10_node-table.up.sql b/internal/repository/migrations/sqlite3/10_node-table.up.sql index caf5589..247bcea 100644 --- a/internal/repository/migrations/sqlite3/10_node-table.up.sql +++ b/internal/repository/migrations/sqlite3/10_node-table.up.sql @@ -27,24 +27,22 @@ CREATE TABLE "node_state" ( FOREIGN KEY (node_id) REFERENCES node (id) ); --- Add NEW Indices For New Job Table Columns -CREATE INDEX IF NOT EXISTS jobs_cluster_submittime ON job (cluster, submit_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_submittime ON job (cluster, cluster_partition, submit_time); -CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_submittime ON job ( - cluster, cluster_partition, job_state, submit_time -); -CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_submittime ON job (cluster, job_state, submit_time); - -- Add NEW Indices For New Node Table VARCHAR Fields CREATE INDEX IF NOT EXISTS nodes_cluster ON node (cluster); CREATE INDEX IF NOT EXISTS nodes_cluster_subcluster ON node (cluster, subcluster); -- Add NEW Indices For New Node_State Table Fields +CREATE INDEX IF NOT EXISTS nodestates_timestamp ON node_state (time_stamp); CREATE INDEX IF NOT EXISTS nodestates_state ON node_state (node_state); CREATE INDEX IF NOT EXISTS nodestates_health ON node_state (health_state); +CREATE INDEX IF NOT EXISTS nodestates_state_timestamp ON node_state (node_state, time_stamp); +CREATE INDEX IF NOT EXISTS nodestates_health_timestamp ON node_state (health_state, time_stamp); CREATE INDEX IF NOT EXISTS nodestates_nodeid_state ON node_state (node_id, node_state); CREATE INDEX IF NOT EXISTS nodestates_nodeid_health ON node_state (node_id, health_state); -- Add NEW Indices For Increased Amounts of Tags CREATE INDEX IF NOT EXISTS tags_jobid ON jobtag (job_id); CREATE INDEX IF NOT EXISTS tags_tagid ON jobtag (tag_id); + +-- Optimize DB index usage +PRAGMA optimize; \ No newline at end of file