ClusterCockpit/cc-backend
1
0
Fork 0
mirror of https://github.com/ClusterCockpit/cc-backend synced 2025-11-04 09:35:07 +01:00
Code Issues Packages Releases Activity

Merge pull request #118 from ClusterCockpit/optimize-sqlite-db-setup

Browse Source 
Add migration for job table contraints
This commit is contained in:
Jan Eitzinger
2023-06-05 16:13:24 +02:00
committed by GitHub
parent 0d4935e244 d53ef526dd
commit e32872181f
9 changed files with 141 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/repository/job_test.go
View File

@@ -53,7 +53,7 @@ func TestGetTags(t *testing.T) {
fmt.Printf("TAGS %+v \n", tags) fmt.Printf("TAGS %+v \n", tags)
// fmt.Printf("COUNTS %+v \n", counts) // fmt.Printf("COUNTS %+v \n", counts)
if counts["bandwidth"] != 3 { if counts["bandwidth"] != 0 {
t.Errorf("wrong tag count \ngot: %d \nwant: 3", counts["bandwidth"]) t.Errorf("wrong tag count \ngot: %d \nwant: 0", counts["bandwidth"])
} }
} }

2
internal/repository/migration.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/golang-migrate/migrate/v4/source/iofs" "github.com/golang-migrate/migrate/v4/source/iofs"
) )
const Version uint = 3 const Version uint = 4
//go:embed migrations/* //go:embed migrations/*
var migrationFiles embed.FS var migrationFiles embed.FS

104
internal/repository/migrations/sqlite3/01_init-schema.up.sql
View File

@@ -1,64 +1,58 @@
CREATE TABLE IF NOT EXISTS job ( CREATE TABLE IF NOT EXISTS job (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
job_id BIGINT NOT NULL, job_id BIGINT NOT NULL,
cluster VARCHAR(255) NOT NULL, cluster VARCHAR(255) NOT NULL,
subcluster VARCHAR(255) NOT NULL, subcluster VARCHAR(255) NOT NULL,
start_time BIGINT NOT NULL, -- Unix timestamp start_time BIGINT NOT NULL, -- Unix timestamp
user VARCHAR(255) NOT NULL,
user VARCHAR(255) NOT NULL, project VARCHAR(255) NOT NULL,
project VARCHAR(255) NOT NULL, partition VARCHAR(255),
partition VARCHAR(255), array_job_id BIGINT,
array_job_id BIGINT, duration INT NOT NULL,
duration INT NOT NULL, walltime INT NOT NULL,
walltime INT NOT NULL, job_state VARCHAR(255) NOT NULL
job_state VARCHAR(255) NOT NULL CHECK(job_state IN ('running', 'completed', 'failed', 'canceled', 'stopped', 'timeout')),
CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', meta_data TEXT, -- JSON
'stopped', 'timeout', 'preempted', 'out_of_memory')), resources TEXT NOT NULL, -- JSON
meta_data TEXT, -- JSON num_nodes INT NOT NULL,
resources TEXT NOT NULL, -- JSON num_hwthreads INT,
num_acc INT,
num_nodes INT NOT NULL, smt TINYINT NOT NULL DEFAULT 1 CHECK(smt IN (0, 1 )),
num_hwthreads INT, exclusive TINYINT NOT NULL DEFAULT 1 CHECK(exclusive IN (0, 1, 2)),
num_acc INT, monitoring_status TINYINT NOT NULL DEFAULT 1 CHECK(monitoring_status IN (0, 1, 2, 3)),
smt TINYINT NOT NULL DEFAULT 1 CHECK(smt IN (0, 1 )), mem_used_max REAL NOT NULL DEFAULT 0.0,
exclusive TINYINT NOT NULL DEFAULT 1 CHECK(exclusive IN (0, 1, 2)), flops_any_avg REAL NOT NULL DEFAULT 0.0,
monitoring_status TINYINT NOT NULL DEFAULT 1 CHECK(monitoring_status IN (0, 1, 2, 3)), mem_bw_avg REAL NOT NULL DEFAULT 0.0,
load_avg REAL NOT NULL DEFAULT 0.0,
mem_used_max REAL NOT NULL DEFAULT 0.0, net_bw_avg REAL NOT NULL DEFAULT 0.0,
flops_any_avg REAL NOT NULL DEFAULT 0.0, net_data_vol_total REAL NOT NULL DEFAULT 0.0,
mem_bw_avg REAL NOT NULL DEFAULT 0.0, file_bw_avg REAL NOT NULL DEFAULT 0.0,
load_avg REAL NOT NULL DEFAULT 0.0, file_data_vol_total REAL NOT NULL DEFAULT 0.0);
net_bw_avg REAL NOT NULL DEFAULT 0.0,
net_data_vol_total REAL NOT NULL DEFAULT 0.0,
file_bw_avg REAL NOT NULL DEFAULT 0.0,
file_data_vol_total REAL NOT NULL DEFAULT 0.0,
UNIQUE (job_id, cluster, start_time)
);
CREATE TABLE IF NOT EXISTS tag ( CREATE TABLE IF NOT EXISTS tag (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
tag_type VARCHAR(255) NOT NULL, tag_type VARCHAR(255) NOT NULL,
tag_name VARCHAR(255) NOT NULL, tag_name VARCHAR(255) NOT NULL,
UNIQUE (tag_type, tag_name)); UNIQUE (tag_type, tag_name));
CREATE TABLE IF NOT EXISTS jobtag ( CREATE TABLE IF NOT EXISTS jobtag (
job_id INTEGER, job_id INTEGER,
tag_id INTEGER, tag_id INTEGER,
PRIMARY KEY (job_id, tag_id), PRIMARY KEY (job_id, tag_id),
FOREIGN KEY (job_id) REFERENCES job (id) ON DELETE CASCADE, FOREIGN KEY (job_id) REFERENCES job (id) ON DELETE CASCADE,
FOREIGN KEY (tag_id) REFERENCES tag (id) ON DELETE CASCADE); FOREIGN KEY (tag_id) REFERENCES tag (id) ON DELETE CASCADE);
CREATE TABLE IF NOT EXISTS configuration ( CREATE TABLE IF NOT EXISTS configuration (
username varchar(255), username varchar(255),
confkey varchar(255), confkey varchar(255),
value varchar(255), value varchar(255),
PRIMARY KEY (username, confkey), PRIMARY KEY (username, confkey),
FOREIGN KEY (username) REFERENCES user (username) ON DELETE CASCADE ON UPDATE NO ACTION); FOREIGN KEY (username) REFERENCES user (username) ON DELETE CASCADE ON UPDATE NO ACTION);
CREATE TABLE IF NOT EXISTS user ( CREATE TABLE IF NOT EXISTS user (
username varchar(255) PRIMARY KEY NOT NULL, username varchar(255) PRIMARY KEY NOT NULL,
password varchar(255) DEFAULT NULL, password varchar(255) DEFAULT NULL,
ldap tinyint NOT NULL DEFAULT 0, /* col called "ldap" for historic reasons, fills the "AuthSource" */ ldap tinyint NOT NULL DEFAULT 0, /* col called "ldap" for historic reasons, fills the "AuthSource" */
name varchar(255) DEFAULT NULL, name varchar(255) DEFAULT NULL,
roles varchar(255) NOT NULL DEFAULT "[]", roles varchar(255) NOT NULL DEFAULT "[]",
email varchar(255) DEFAULT NULL); email varchar(255) DEFAULT NULL);

44
internal/repository/migrations/sqlite3/04_add-constraints.down.sql Normal file
View File

@@ -0,0 +1,44 @@
CREATE TABLE IF NOT EXISTS job_new (
id INTEGER PRIMARY KEY,
job_id BIGINT NOT NULL,
cluster VARCHAR(255) NOT NULL,
subcluster VARCHAR(255) NOT NULL,
start_time BIGINT NOT NULL, -- Unix timestamp
user VARCHAR(255) NOT NULL,
project VARCHAR(255) NOT NULL,
partition VARCHAR(255),
array_job_id BIGINT,
duration INT NOT NULL,
walltime INT NOT NULL,
job_state VARCHAR(255) NOT NULL
CHECK(job_state IN ('running', 'completed', 'failed', 'canceled', 'stopped', 'timeout')),
meta_data TEXT, -- JSON
resources TEXT NOT NULL, -- JSON
num_nodes INT NOT NULL,
num_hwthreads INT,
num_acc INT,
smt TINYINT NOT NULL DEFAULT 1 CHECK(smt IN (0, 1 )),
exclusive TINYINT NOT NULL DEFAULT 1 CHECK(exclusive IN (0, 1, 2)),
monitoring_status TINYINT NOT NULL DEFAULT 1 CHECK(monitoring_status IN (0, 1, 2, 3)),
mem_used_max REAL NOT NULL DEFAULT 0.0,
flops_any_avg REAL NOT NULL DEFAULT 0.0,
mem_bw_avg REAL NOT NULL DEFAULT 0.0,
load_avg REAL NOT NULL DEFAULT 0.0,
net_bw_avg REAL NOT NULL DEFAULT 0.0,
net_data_vol_total REAL NOT NULL DEFAULT 0.0,
file_bw_avg REAL NOT NULL DEFAULT 0.0,
file_data_vol_total REAL NOT NULL DEFAULT 0.0;
INSERT INTO job_new SELECT * FROM job;
DROP TABLE job;
ALTER TABLE job_new RENAME TO job;
CREATE INDEX IF NOT EXISTS job_stats ON job (cluster,subcluster,user);
CREATE INDEX IF NOT EXISTS job_by_user ON job (user);
CREATE INDEX IF NOT EXISTS job_by_starttime ON job (start_time);
CREATE INDEX IF NOT EXISTS job_by_job_id ON job (job_id, cluster, start_time);
CREATE INDEX IF NOT EXISTS job_list ON job (cluster, job_state);
CREATE INDEX IF NOT EXISTS job_list_user ON job (user, cluster, job_state);
CREATE INDEX IF NOT EXISTS job_list_users ON job (user, job_state);
CREATE INDEX IF NOT EXISTS job_list_users_start ON job (start_time, user, job_state);

44
internal/repository/migrations/sqlite3/04_add-constraints.up.sql Normal file
View File

@@ -0,0 +1,44 @@
CREATE TABLE IF NOT EXISTS job_new (
id INTEGER PRIMARY KEY,
job_id BIGINT NOT NULL,
cluster VARCHAR(255) NOT NULL,
subcluster VARCHAR(255) NOT NULL,
start_time BIGINT NOT NULL, -- Unix timestamp
user VARCHAR(255) NOT NULL,
project VARCHAR(255) NOT NULL,
partition VARCHAR(255),
array_job_id BIGINT,
duration INT NOT NULL,
walltime INT NOT NULL,
job_state VARCHAR(255) NOT NULL
CHECK(job_state IN ('running', 'completed', 'failed', 'cancelled', 'stopped', 'timeout', 'preempted', 'out_of_memory')),
meta_data TEXT, -- JSON
resources TEXT NOT NULL, -- JSON
num_nodes INT NOT NULL,
num_hwthreads INT,
num_acc INT,
smt TINYINT NOT NULL DEFAULT 1 CHECK(smt IN (0, 1 )),
exclusive TINYINT NOT NULL DEFAULT 1 CHECK(exclusive IN (0, 1, 2)),
monitoring_status TINYINT NOT NULL DEFAULT 1 CHECK(monitoring_status IN (0, 1, 2, 3)),
mem_used_max REAL NOT NULL DEFAULT 0.0,
flops_any_avg REAL NOT NULL DEFAULT 0.0,
mem_bw_avg REAL NOT NULL DEFAULT 0.0,
load_avg REAL NOT NULL DEFAULT 0.0,
net_bw_avg REAL NOT NULL DEFAULT 0.0,
net_data_vol_total REAL NOT NULL DEFAULT 0.0,
file_bw_avg REAL NOT NULL DEFAULT 0.0,
file_data_vol_total REAL NOT NULL DEFAULT 0.0,
UNIQUE (job_id, cluster, start_time));
INSERT INTO job_new SELECT * FROM job;
DROP TABLE job;
ALTER TABLE job_new RENAME TO job;
CREATE INDEX IF NOT EXISTS job_stats ON job (cluster,subcluster,user);
CREATE INDEX IF NOT EXISTS job_by_user ON job (user);
CREATE INDEX IF NOT EXISTS job_by_starttime ON job (start_time);
CREATE INDEX IF NOT EXISTS job_by_job_id ON job (job_id, cluster, start_time);
CREATE INDEX IF NOT EXISTS job_list ON job (cluster, job_state);
CREATE INDEX IF NOT EXISTS job_list_user ON job (user, cluster, job_state);
CREATE INDEX IF NOT EXISTS job_list_users ON job (user, job_state);
CREATE INDEX IF NOT EXISTS job_list_users_start ON job (start_time, user, job_state);

BIN
internal/repository/testdata/job.db-shm vendored
View File

Binary file not shown.

0
internal/repository/testdata/job.db-wal vendored
View File

BIN
internal/repository/testdata/test.db vendored
View File

Binary file not shown.

2
internal/repository/user_test.go
View File

@@ -34,7 +34,7 @@ func setupUserTest(t *testing.T) *UserCfgRepo {
}` }`
log.Init("info", true) log.Init("info", true)
dbfilepath := "testdata/test.db" dbfilepath := "testdata/job.db"
err := MigrateDB("sqlite3", dbfilepath) err := MigrateDB("sqlite3", dbfilepath)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)