ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2024-11-14 02:37:25 +01:00
Code Issues Packages Releases Activity

Merge pull request #118 from ClusterCockpit/optimize-sqlite-db-setup

Browse Source 
Add migration for job table contraints
This commit is contained in:
Jan Eitzinger 2023-06-05 16:13:24 +02:00 committed by GitHub
commit e32872181f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 141 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/repository/job_test.go
View File

@ -53,7 +53,7 @@ func TestGetTags(t *testing.T) {
fmt.Printf("TAGS %+v \n", tags) fmt.Printf("TAGS %+v \n", tags)
// fmt.Printf("COUNTS %+v \n", counts) // fmt.Printf("COUNTS %+v \n", counts)
if counts["bandwidth"] != 3 { if counts["bandwidth"] != 0 {
t.Errorf("wrong tag count \ngot: %d \nwant: 3", counts["bandwidth"]) t.Errorf("wrong tag count \ngot: %d \nwant: 0", counts["bandwidth"])
} }
} }

2
internal/repository/migration.go
View File

@ -16,7 +16,7 @@ import (
"github.com/golang-migrate/migrate/v4/source/iofs" "github.com/golang-migrate/migrate/v4/source/iofs"
) )
const Version uint = 3 const Version uint = 4
//go:embed migrations/* //go:embed migrations/*
var migrationFiles embed.FS var migrationFiles embed.FS

10
internal/repository/migrations/sqlite3/01_init-schema.up.sql
View File

@ -4,7 +4,6 @@ CREATE TABLE IF NOT EXISTS job (
cluster VARCHAR(255) NOT NULL, cluster VARCHAR(255) NOT NULL,
subcluster VARCHAR(255) NOT NULL, subcluster VARCHAR(255) NOT NULL,
start_time BIGINT NOT NULL, -- Unix timestamp start_time BIGINT NOT NULL, -- Unix timestamp
user VARCHAR(255) NOT NULL, user VARCHAR(255) NOT NULL,
project VARCHAR(255) NOT NULL, project VARCHAR(255) NOT NULL,
partition VARCHAR(255), partition VARCHAR(255),
@ -12,18 +11,15 @@ CREATE TABLE IF NOT EXISTS job (
duration INT NOT NULL, duration INT NOT NULL,
walltime INT NOT NULL, walltime INT NOT NULL,
job_state VARCHAR(255) NOT NULL job_state VARCHAR(255) NOT NULL
CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', CHECK(job_state IN ('running', 'completed', 'failed', 'canceled', 'stopped', 'timeout')),
'stopped', 'timeout', 'preempted', 'out_of_memory')),
meta_data TEXT, -- JSON meta_data TEXT, -- JSON
resources TEXT NOT NULL, -- JSON resources TEXT NOT NULL, -- JSON
num_nodes INT NOT NULL, num_nodes INT NOT NULL,
num_hwthreads INT, num_hwthreads INT,
num_acc INT, num_acc INT,
smt TINYINT NOT NULL DEFAULT 1 CHECK(smt IN (0, 1 )), smt TINYINT NOT NULL DEFAULT 1 CHECK(smt IN (0, 1 )),
exclusive TINYINT NOT NULL DEFAULT 1 CHECK(exclusive IN (0, 1, 2)), exclusive TINYINT NOT NULL DEFAULT 1 CHECK(exclusive IN (0, 1, 2)),
monitoring_status TINYINT NOT NULL DEFAULT 1 CHECK(monitoring_status IN (0, 1, 2, 3)), monitoring_status TINYINT NOT NULL DEFAULT 1 CHECK(monitoring_status IN (0, 1, 2, 3)),
mem_used_max REAL NOT NULL DEFAULT 0.0, mem_used_max REAL NOT NULL DEFAULT 0.0,
flops_any_avg REAL NOT NULL DEFAULT 0.0, flops_any_avg REAL NOT NULL DEFAULT 0.0,
mem_bw_avg REAL NOT NULL DEFAULT 0.0, mem_bw_avg REAL NOT NULL DEFAULT 0.0,
@ -31,9 +27,7 @@ CREATE TABLE IF NOT EXISTS job (
net_bw_avg REAL NOT NULL DEFAULT 0.0, net_bw_avg REAL NOT NULL DEFAULT 0.0,
net_data_vol_total REAL NOT NULL DEFAULT 0.0, net_data_vol_total REAL NOT NULL DEFAULT 0.0,
file_bw_avg REAL NOT NULL DEFAULT 0.0, file_bw_avg REAL NOT NULL DEFAULT 0.0,
file_data_vol_total REAL NOT NULL DEFAULT 0.0, file_data_vol_total REAL NOT NULL DEFAULT 0.0);
UNIQUE (job_id, cluster, start_time)
);
CREATE TABLE IF NOT EXISTS tag ( CREATE TABLE IF NOT EXISTS tag (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,

44
internal/repository/migrations/sqlite3/04_add-constraints.down.sql Normal file
View File

@ -0,0 +1,44 @@
CREATE TABLE IF NOT EXISTS job_new (
id INTEGER PRIMARY KEY,
job_id BIGINT NOT NULL,
cluster VARCHAR(255) NOT NULL,
subcluster VARCHAR(255) NOT NULL,
start_time BIGINT NOT NULL, -- Unix timestamp
user VARCHAR(255) NOT NULL,
project VARCHAR(255) NOT NULL,
partition VARCHAR(255),
array_job_id BIGINT,
duration INT NOT NULL,
walltime INT NOT NULL,
job_state VARCHAR(255) NOT NULL
CHECK(job_state IN ('running', 'completed', 'failed', 'canceled', 'stopped', 'timeout')),
meta_data TEXT, -- JSON
resources TEXT NOT NULL, -- JSON
num_nodes INT NOT NULL,
num_hwthreads INT,
num_acc INT,
smt TINYINT NOT NULL DEFAULT 1 CHECK(smt IN (0, 1 )),
exclusive TINYINT NOT NULL DEFAULT 1 CHECK(exclusive IN (0, 1, 2)),
monitoring_status TINYINT NOT NULL DEFAULT 1 CHECK(monitoring_status IN (0, 1, 2, 3)),
mem_used_max REAL NOT NULL DEFAULT 0.0,
flops_any_avg REAL NOT NULL DEFAULT 0.0,
mem_bw_avg REAL NOT NULL DEFAULT 0.0,
load_avg REAL NOT NULL DEFAULT 0.0,
net_bw_avg REAL NOT NULL DEFAULT 0.0,
net_data_vol_total REAL NOT NULL DEFAULT 0.0,
file_bw_avg REAL NOT NULL DEFAULT 0.0,
file_data_vol_total REAL NOT NULL DEFAULT 0.0;
INSERT INTO job_new SELECT * FROM job;
DROP TABLE job;
ALTER TABLE job_new RENAME TO job;
CREATE INDEX IF NOT EXISTS job_stats ON job (cluster,subcluster,user);
CREATE INDEX IF NOT EXISTS job_by_user ON job (user);
CREATE INDEX IF NOT EXISTS job_by_starttime ON job (start_time);
CREATE INDEX IF NOT EXISTS job_by_job_id ON job (job_id, cluster, start_time);
CREATE INDEX IF NOT EXISTS job_list ON job (cluster, job_state);
CREATE INDEX IF NOT EXISTS job_list_user ON job (user, cluster, job_state);
CREATE INDEX IF NOT EXISTS job_list_users ON job (user, job_state);
CREATE INDEX IF NOT EXISTS job_list_users_start ON job (start_time, user, job_state);

44
internal/repository/migrations/sqlite3/04_add-constraints.up.sql Normal file
View File

@ -0,0 +1,44 @@
CREATE TABLE IF NOT EXISTS job_new (
id INTEGER PRIMARY KEY,
job_id BIGINT NOT NULL,
cluster VARCHAR(255) NOT NULL,
subcluster VARCHAR(255) NOT NULL,
start_time BIGINT NOT NULL, -- Unix timestamp
user VARCHAR(255) NOT NULL,
project VARCHAR(255) NOT NULL,
partition VARCHAR(255),
array_job_id BIGINT,
duration INT NOT NULL,
walltime INT NOT NULL,
job_state VARCHAR(255) NOT NULL
CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', 'stopped', 'timeout', 'preempted', 'out_of_memory')),
meta_data TEXT, -- JSON
resources TEXT NOT NULL, -- JSON
num_nodes INT NOT NULL,
num_hwthreads INT,
num_acc INT,
smt TINYINT NOT NULL DEFAULT 1 CHECK(smt IN (0, 1 )),
exclusive TINYINT NOT NULL DEFAULT 1 CHECK(exclusive IN (0, 1, 2)),
monitoring_status TINYINT NOT NULL DEFAULT 1 CHECK(monitoring_status IN (0, 1, 2, 3)),
mem_used_max REAL NOT NULL DEFAULT 0.0,
flops_any_avg REAL NOT NULL DEFAULT 0.0,
mem_bw_avg REAL NOT NULL DEFAULT 0.0,
load_avg REAL NOT NULL DEFAULT 0.0,
net_bw_avg REAL NOT NULL DEFAULT 0.0,
net_data_vol_total REAL NOT NULL DEFAULT 0.0,
file_bw_avg REAL NOT NULL DEFAULT 0.0,
file_data_vol_total REAL NOT NULL DEFAULT 0.0,
UNIQUE (job_id, cluster, start_time));
INSERT INTO job_new SELECT * FROM job;
DROP TABLE job;
ALTER TABLE job_new RENAME TO job;
CREATE INDEX IF NOT EXISTS job_stats ON job (cluster,subcluster,user);
CREATE INDEX IF NOT EXISTS job_by_user ON job (user);
CREATE INDEX IF NOT EXISTS job_by_starttime ON job (start_time);
CREATE INDEX IF NOT EXISTS job_by_job_id ON job (job_id, cluster, start_time);
CREATE INDEX IF NOT EXISTS job_list ON job (cluster, job_state);
CREATE INDEX IF NOT EXISTS job_list_user ON job (user, cluster, job_state);
CREATE INDEX IF NOT EXISTS job_list_users ON job (user, job_state);
CREATE INDEX IF NOT EXISTS job_list_users_start ON job (start_time, user, job_state);

BIN
internal/repository/testdata/job.db-shm vendored
View File

Binary file not shown.

0
internal/repository/testdata/job.db-wal vendored
View File

BIN
internal/repository/testdata/test.db vendored
View File

Binary file not shown.

2
internal/repository/user_test.go
View File

@ -34,7 +34,7 @@ func setupUserTest(t *testing.T) *UserCfgRepo {
}` }`
log.Init("info", true) log.Init("info", true)
dbfilepath := "testdata/test.db" dbfilepath := "testdata/job.db"
err := MigrateDB("sqlite3", dbfilepath) err := MigrateDB("sqlite3", dbfilepath)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)