From 0f87f651f99b8f24f59fb75dbb463fff0bed5560 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 1 Jun 2023 08:03:12 +0200 Subject: [PATCH 1/8] Refactor DB connection settings --- internal/repository/dbConnection.go | 55 +++++++++++++++++++---------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/internal/repository/dbConnection.go b/internal/repository/dbConnection.go index 790ec74..26154d5 100644 --- a/internal/repository/dbConnection.go +++ b/internal/repository/dbConnection.go @@ -5,15 +5,11 @@ package repository import ( - "database/sql" - "fmt" "log" "sync" "time" "github.com/jmoiron/sqlx" - "github.com/mattn/go-sqlite3" - "github.com/qustavo/sqlhooks/v2" ) var ( @@ -26,35 +22,56 @@ type DBConnection struct { Driver string } +type DatabaseOptions struct { + URL string + MaxOpenConnections int + MaxIdleConnections int + ConnectionMaxLifetime time.Duration + ConnectionMaxIdleTime time.Duration +} + func Connect(driver string, db string) { var err error var dbHandle *sqlx.DB dbConnOnce.Do(func() { - if driver == "sqlite3" { - sql.Register("sqlite3WithHooks", sqlhooks.Wrap(&sqlite3.SQLiteDriver{}, &Hooks{})) - dbHandle, err = sqlx.Open("sqlite3WithHooks", fmt.Sprintf("%s?_foreign_keys=on", db)) - // dbHandle, err = sqlx.Open("sqlite3", fmt.Sprintf("%s?_foreign_keys=on", db)) + opts := DatabaseOptions{ + URL: db, + MaxOpenConnections: 5, + MaxIdleConnections: 5, + ConnectionMaxLifetime: time.Hour, + ConnectionMaxIdleTime: time.Hour, + } + + switch driver { + case "sqlite3": + // sql.Register("sqlite3WithHooks", sqlhooks.Wrap(&sqlite3.SQLiteDriver{}, &Hooks{})) + + // - Set WAL mode (not strictly necessary each time because it's persisted in the database, but good for first run) + // - Set busy timeout, so concurrent writers wait on each other instead of erroring immediately + // - Enable foreign key checks + opts.URL += "?_journal=WAL&_timeout=5000&_fk=true" + + // dbHandle, err = sqlx.Open("sqlite3WithHooks", fmt.Sprintf("%s?_foreign_keys=on", db)) + dbHandle, err = sqlx.Open("sqlite3", opts.URL) if err != nil { log.Fatal(err) } - - // sqlite does not multithread. Having more than one connection open would just mean - // waiting for locks. - dbHandle.SetMaxOpenConns(1) - } else if driver == "mysql" { - dbHandle, err = sqlx.Open("mysql", fmt.Sprintf("%s?multiStatements=true", db)) + case "mysql": + opts.URL += "?multiStatements=true" + dbHandle, err = sqlx.Open("mysql", opts.URL) if err != nil { log.Fatalf("sqlx.Open() error: %v", err) } - - dbHandle.SetConnMaxLifetime(time.Minute * 3) - dbHandle.SetMaxOpenConns(10) - dbHandle.SetMaxIdleConns(10) - } else { + default: log.Fatalf("unsupported database driver: %s", driver) } + dbHandle.SetMaxOpenConns(opts.MaxOpenConnections) + dbHandle.SetMaxIdleConns(opts.MaxIdleConnections) + dbHandle.SetConnMaxLifetime(opts.ConnectionMaxLifetime) + dbHandle.SetConnMaxIdleTime(opts.ConnectionMaxIdleTime) + dbConnInstance = &DBConnection{DB: dbHandle, Driver: driver} err = checkDBVersion(driver, dbHandle.DB) if err != nil { From 0a472d2b395b37dbe3d7c05f7dfdf763fb67a53f Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 1 Jun 2023 10:12:45 +0200 Subject: [PATCH 2/8] Fix bug in logger --- pkg/log/log.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/log/log.go b/pkg/log/log.go index 8240194..f514df9 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -44,6 +44,7 @@ var ( /* CONFIG */ func Init(lvl string, logdate bool) { + switch lvl { case "crit": ErrWriter = io.Discard @@ -70,6 +71,12 @@ func Init(lvl string, logdate bool) { WarnLog = log.New(WarnWriter, WarnPrefix, log.Lshortfile) ErrLog = log.New(ErrWriter, ErrPrefix, log.Llongfile) CritLog = log.New(CritWriter, CritPrefix, log.Llongfile) + } else { + DebugLog = log.New(DebugWriter, DebugPrefix, log.LstdFlags) + InfoLog = log.New(InfoWriter, InfoPrefix, log.LstdFlags|log.Lshortfile) + WarnLog = log.New(WarnWriter, WarnPrefix, log.LstdFlags|log.Lshortfile) + ErrLog = log.New(ErrWriter, ErrPrefix, log.LstdFlags|log.Llongfile) + CritLog = log.New(CritWriter, CritPrefix, log.LstdFlags|log.Llongfile) } } From 012b131b232e786617debda58776d8a842a9ef24 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 1 Jun 2023 13:57:35 +0200 Subject: [PATCH 3/8] Refactor for benchmarking --- internal/repository/dbConnection.go | 4 +- internal/repository/job.go | 2 +- internal/repository/job_test.go | 12 --- internal/repository/query.go | 69 ++++++++++---- internal/repository/repository_bench.go | 119 ++++++++++++++++++++++++ 5 files changed, 175 insertions(+), 31 deletions(-) create mode 100644 internal/repository/repository_bench.go diff --git a/internal/repository/dbConnection.go b/internal/repository/dbConnection.go index 26154d5..da3d40e 100644 --- a/internal/repository/dbConnection.go +++ b/internal/repository/dbConnection.go @@ -37,8 +37,8 @@ func Connect(driver string, db string) { dbConnOnce.Do(func() { opts := DatabaseOptions{ URL: db, - MaxOpenConnections: 5, - MaxIdleConnections: 5, + MaxOpenConnections: 4, + MaxIdleConnections: 4, ConnectionMaxLifetime: time.Hour, ConnectionMaxIdleTime: time.Hour, } diff --git a/internal/repository/job.go b/internal/repository/job.go index 4eee0d3..c5df610 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -74,7 +74,7 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { &job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId, &job.NumNodes, &job.NumHWThreads, &job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State, &job.Duration, &job.Walltime, &job.RawResources /*&job.RawMetaData*/); err != nil { - log.Warn("Error while scanning rows") + log.Warnf("Error while scanning rows: %v", err) return nil, err } diff --git a/internal/repository/job_test.go b/internal/repository/job_test.go index d74dad5..bf491f4 100644 --- a/internal/repository/job_test.go +++ b/internal/repository/job_test.go @@ -8,21 +8,9 @@ import ( "fmt" "testing" - "github.com/ClusterCockpit/cc-backend/pkg/log" _ "github.com/mattn/go-sqlite3" ) -func setup(t *testing.T) *JobRepository { - log.Init("info", true) - dbfilepath := "testdata/test.db" - err := MigrateDB("sqlite3", dbfilepath) - if err != nil { - t.Fatal(err) - } - Connect("sqlite3", dbfilepath) - return GetJobRepository() -} - func TestFind(t *testing.T) { r := setup(t) diff --git a/internal/repository/query.go b/internal/repository/query.go index 60354c1..5513d51 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -19,19 +19,12 @@ import ( sq "github.com/Masterminds/squirrel" ) -// QueryJobs returns a list of jobs matching the provided filters. page and order are optional- -func (r *JobRepository) QueryJobs( - ctx context.Context, +func (r *JobRepository) queryJobs( + query sq.SelectBuilder, filters []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) ([]*schema.Job, error) { - query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job")) - - if qerr != nil { - return nil, qerr - } - if order != nil { field := toSnakeCase(order.Field) @@ -81,21 +74,46 @@ func (r *JobRepository) QueryJobs( return jobs, nil } -// CountJobs counts the number of jobs matching the filters. -func (r *JobRepository) CountJobs( - ctx context.Context, - filters []*model.JobFilter) (int, error) { +func (r *JobRepository) testQueryJobs( + filters []*model.JobFilter, + page *model.PageRequest, + order *model.OrderByInput) ([]*schema.Job, error) { - // count all jobs: - query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job")) + return r.queryJobs(sq.Select(jobColumns...).From("job"), + filters, page, order) +} + +// QueryJobs returns a list of jobs matching the provided filters. page and order are optional- +func (r *JobRepository) QueryJobs( + ctx context.Context, + filters []*model.JobFilter, + page *model.PageRequest, + order *model.OrderByInput) ([]*schema.Job, error) { + + query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job")) if qerr != nil { - return 0, qerr + return nil, qerr } + return r.queryJobs(query, + filters, page, order) +} + +func (r *JobRepository) countJobs(query sq.SelectBuilder, + filters []*model.JobFilter) (int, error) { + for _, f := range filters { query = BuildWhereClause(f, query) } + + sql, args, err := query.ToSql() + if err != nil { + log.Warn("Error while converting query to sql") + return 0, nil + } + + log.Debugf("SQL query: `%s`, args: %#v", sql, args) var count int if err := query.RunWith(r.DB).Scan(&count); err != nil { return 0, err @@ -104,6 +122,25 @@ func (r *JobRepository) CountJobs( return count, nil } +func (r *JobRepository) testCountJobs( + filters []*model.JobFilter) (int, error) { + + return r.countJobs(sq.Select("count(*)").From("job"), filters) +} + +func (r *JobRepository) CountJobs( + ctx context.Context, + filters []*model.JobFilter) (int, error) { + + query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job")) + + if qerr != nil { + return 0, qerr + } + + return r.countJobs(query, filters) +} + func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (queryOut sq.SelectBuilder, err error) { user := auth.GetUser(ctx) if user == nil || user.HasAnyRole([]auth.Role{auth.RoleAdmin, auth.RoleSupport, auth.RoleApi}) { // Admin & Co. : All jobs diff --git a/internal/repository/repository_bench.go b/internal/repository/repository_bench.go new file mode 100644 index 0000000..598d379 --- /dev/null +++ b/internal/repository/repository_bench.go @@ -0,0 +1,119 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package repository + +import ( + "testing" + + "github.com/ClusterCockpit/cc-backend/internal/graph/model" + "github.com/ClusterCockpit/cc-backend/pkg/log" + _ "github.com/mattn/go-sqlite3" +) + +func TestPragma(t *testing.T) { + t.Run("sets up a new DB", func(t *testing.T) { + db := setup(t) + + for _, pragma := range []string{"synchronous", "journal_mode", "busy_timeout", "auto_vacuum", "foreign_keys"} { + t.Log("PRAGMA", pragma, getPragma(db, pragma)) + } + }) +} + +func getPragma(db *JobRepository, name string) string { + var s string + if err := db.DB.QueryRow(`PRAGMA ` + name).Scan(&s); err != nil { + panic(err) + } + return s +} + +func BenchmarkSelect1(b *testing.B) { + db := setup(b) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := db.DB.Exec(`select 1`) + noErr(b, err) + } + }) +} + +func BenchmarkDB_FindJobById(b *testing.B) { + var jobId int64 = 1677322 + + b.Run("FindJobById", func(b *testing.B) { + db := setup(b) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := db.FindById(jobId) + noErr(b, err) + } + }) + }) +} + +func BenchmarkDB_FindJob(b *testing.B) { + var jobId int64 = 107266 + var startTime int64 = 1657557241 + var cluster = "fritz" + + b.Run("FindJob", func(b *testing.B) { + db := setup(b) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := db.Find(&jobId, &cluster, &startTime) + noErr(b, err) + } + }) + }) +} + +func BenchmarkDB_CountJobs(b *testing.B) { + filter := &model.JobFilter{} + filter.State = append(filter.State, "running") + cluster := "fritz" + filter.Cluster = &model.StringInput{Eq: &cluster} + + b.Run("CountJobs", func(b *testing.B) { + db := setup(b) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := db.testCountJobs([]*model.JobFilter{filter}) + noErr(b, err) + } + }) + }) +} + +func setup(tb testing.TB) *JobRepository { + tb.Helper() + log.Init("warn", true) + dbfile := "testdata/job.db" + err := MigrateDB("sqlite3", dbfile) + noErr(tb, err) + + Connect("sqlite3", dbfile) + return GetJobRepository() +} + +func noErr(tb testing.TB, err error) { + tb.Helper() + + if err != nil { + tb.Fatal("Error is not nil:", err) + } +} From 205a400c6fb97d9635db67b105cf161551b016b9 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 1 Jun 2023 14:10:10 +0200 Subject: [PATCH 4/8] Also Benchmarks need _test in name --- internal/repository/{repository_bench.go => repository_test.go} | 2 ++ 1 file changed, 2 insertions(+) rename internal/repository/{repository_bench.go => repository_test.go} (97%) diff --git a/internal/repository/repository_bench.go b/internal/repository/repository_test.go similarity index 97% rename from internal/repository/repository_bench.go rename to internal/repository/repository_test.go index 598d379..0aec9f6 100644 --- a/internal/repository/repository_bench.go +++ b/internal/repository/repository_test.go @@ -84,6 +84,8 @@ func BenchmarkDB_CountJobs(b *testing.B) { filter.State = append(filter.State, "running") cluster := "fritz" filter.Cluster = &model.StringInput{Eq: &cluster} + user := "mppi133h" + filter.User = &model.StringInput{Eq: &user} b.Run("CountJobs", func(b *testing.B) { db := setup(b) From 65cebf6fde72dd1a8235d892654c964c30a8fffc Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 1 Jun 2023 15:26:53 +0200 Subject: [PATCH 5/8] Add fix for broken ids in job archives --- pkg/schema/cluster.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/schema/cluster.go b/pkg/schema/cluster.go index 0724ada..bc7a86a 100644 --- a/pkg/schema/cluster.go +++ b/pkg/schema/cluster.go @@ -162,10 +162,13 @@ func (topo *Topology) GetMemoryDomainsFromHWThreads( // Temporary fix to convert back from int id to string id for accelerators func (topo *Topology) GetAcceleratorID(id int) (string, error) { - if id < len(topo.Accelerators) { + if id < 0 { + fmt.Printf("ID smaller than 0!\n") + return topo.Accelerators[0].ID, nil + } else if id < len(topo.Accelerators) { return topo.Accelerators[id].ID, nil } else { - return "", fmt.Errorf("Index %d out of range", id) + return "", fmt.Errorf("index %d out of range", id) } } From 447fc574e8cdde2d6d494f9e5794071d89f80d36 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 1 Jun 2023 15:27:19 +0200 Subject: [PATCH 6/8] Add query benchmark --- internal/repository/repository_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 0aec9f6..efb5395 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -101,6 +101,30 @@ func BenchmarkDB_CountJobs(b *testing.B) { }) } +func BenchmarkDB_QueryJobs(b *testing.B) { + filter := &model.JobFilter{} + filter.State = append(filter.State, "running") + cluster := "fritz" + filter.Cluster = &model.StringInput{Eq: &cluster} + user := "mppi133h" + filter.User = &model.StringInput{Eq: &user} + page := &model.PageRequest{ItemsPerPage: 50, Page: 1} + order := &model.OrderByInput{Field: "startTime", Order: model.SortDirectionEnumDesc} + + b.Run("QueryJobs", func(b *testing.B) { + db := setup(b) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := db.testQueryJobs([]*model.JobFilter{filter}, page, order) + noErr(b, err) + } + }) + }) +} + func setup(tb testing.TB) *JobRepository { tb.Helper() log.Init("warn", true) From 203ccdf73b039bc0494980d92c7d0656b160f987 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 1 Jun 2023 15:29:25 +0200 Subject: [PATCH 7/8] Rename test db --- .gitignore | 2 ++ internal/repository/testdata/job.db | Bin 0 -> 81920 bytes internal/repository/testdata/job.db-shm | Bin 0 -> 32768 bytes internal/repository/testdata/job.db-wal | 0 internal/repository/testdata/test.db | Bin 81920 -> 73728 bytes 5 files changed, 2 insertions(+) create mode 100644 internal/repository/testdata/job.db create mode 100644 internal/repository/testdata/job.db-shm create mode 100644 internal/repository/testdata/job.db-wal diff --git a/.gitignore b/.gitignore index 153e354..9f448aa 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ /.vscode/* /archive-migration /archive-manager +var/job.db-shm +var/job.db-wal diff --git a/internal/repository/testdata/job.db b/internal/repository/testdata/job.db new file mode 100644 index 0000000000000000000000000000000000000000..c07fb326cf5522b266cc64674df3890a2bc607c8 GIT binary patch literal 81920 zcmeI5TWlLwdVqP8L@F0LQ4}THIG!-cO4^Jx7v3bM6I+sLM~O_yB9%)mV=^3$$g$@_ z&&8592)a&?T@-%sLz)&TnCDfueGUr7!Qj0_KlzZ-U>s;0QMVv*NvbIZk4bcMPoS#m?N#`tV@oLX9*KQ*)bJoR+(d8)Knp-L+Y3-n;3DcSbAY1O)g zj~7qOtSq#Jw3^hU$l<8QrfN9ebz2J+NV7D>e2bDzjglozQ)*O-ButU!q#CO2n3gJ2 zmLl1vVbj!_s%bXmG!!Z|<2sG9=~}XqqQ1opCcM&{ZyHKe! zXH!s5MVC}#9*xHqMzT{bU7Bx%Nrcl3`uJQS8YHIgm{%I@4rnb!@wDoR4epS^ux z7}WmPZ1I2`gC&%U66l@=w}@w=Hw+Ck*V`|U3e#0N%3N5ik2@>)YSjWxCIT9TuhM%PWE z)#|?O>WbRch`~4}g6EaZZqoqa7bVSANSEm}84%uS_QpsFiEW|5NO@|;LE^;Xa&i7- z$uo>?iE(PVc%rymEX@_mEyEfl9w&q3cyXZ!hIekJJU4T^NJ3VYjuTT{TBK&?D)Wn_ z@##eW$f?oYkabm9jNE^J_?4$Uh%iPRsqViIbYb(ufETGb+K8Yg+aP)VO>c4L^S0K0uQ383yCk$1q6bqF)%jUN$ilmrMVL*1nrW3{IP-lk)!wT zMh!A1>Nx)#zf_D3j0_EhU%udRf++aw>muZ3kk>1hv{xy4esvOEL%P*p{A4fO(a&6U#p>6IciM#@hz@sSf%f`cDQLoxprJ#ENk62f=y%y@OcB}SO&_TA>9}mERFmtSu$jWEZs)^q6ftp zg~eO5>C2E}nyy0=QMxrEH z-YN-c7C*|?TT9$_WM#hee5dyI@{*%`)Z3E~8BM3CX3p-XRak_DQ0dQms3|zDW$l`MkN1B&7R_7fnrcOh*DIVxNM6 zW25wDb1g}Fve`AsxE>i88E3+mlHO|UqGH)#o?7#aEjPf`5p}ihuet1KYhGy0KfX`W zorX-~+tN{EEunEWJTSsQcE0bqO>kR4q-!SH7Ts1%x!_9XGIUL~oK31~n%X#V>ch$Z z4Z$BCNB{{S0VIF~kN^@u0!RP}AOR$R1dzZjM<5oChU3ves8=}on-Kirfdr5M5-atyxkgrq4OQihe^fAQ0U4EHi0%>NAORg16y<_;hK~XcDUChN|GJ*BH6+!*^Xt3HLvi^O{W3-Yng_r zE6loS);1XehzV#hbxDUkZDfzFrGOqt>Vh8F^R})!4N=#aKpWFgY@2Z`NhKRyTl`u9 zB8d}WTU-;8n1(6BmTFLTak9Ay*fg^)!bY@KA#BNUEu*ItUM4k7^ee|egV=t4hO+MM z5oCy3X0|M5IbJNC5c90JCzylQvo+V!=Z!TJT5r{O%B;@Ugj{}ldU}e28`9`PPm`~b z4aY06tr;W(5M$|lo=fM)sgsuJHX(w`=D4gUIU~!aYd8V{N1a|Q3kLZPokn?)%L?p3 zcNO7;q}d9!Og2tijxDg%Ea_$^G^s9BEZC#&30I)i7gSw!1TK?-W`|;w-74IZ2!Ut$ z^aRUKuu~N-TgdQ*^pq#E6l{|gpilUUO0YL~0lxMnNG#81JgJhalTcgWD7c(kT7jz^ zxvY>L3ZW#3AvX~2r|@Ey6pN%Qluh}(;uPjgOA)w2$ut!2&6BJw7t7E;E(p-01&)Rt zr9wKNWlzy$L_k0lzHbJd0RHn|e_XW$zvmxpr}Hu>JGG)^3BmOu49)A27qe}=r(QbqVya=< z4*7xu^m(rfz&L2{#ZDE@urx>Wbehi4Svp7O=_z`e<{*%xIiBXyG?$^d>{&Vmo3s^8 zf$glOWka$QX$f)^SPtycB%3<>(%D1Nqi?-^`omWreEV4ZSMOX|9sBe$AMP>DFF(+0 znuRUXd^H$jfoZ1DG=tT?X@G^aiDfTlTEY#qF5?&)-a zIleBYxhJn{GtFcibed+4SYTp;i4Eqm@XEm}53ebBftlvPPV>Z2gB%ZXJjn4N$AcW7 z-^*whJB{|Sjn|lO`~WE zfoL?ENW>!W&HsM#`+tcT`TRc$oBxv^!4rUA-uxBB8Xy5AfCP{L52~@&~;Y)q-(2>wxHAOcc;iX47_L1Z4NsZYkT{!owTUt4n zPp3;4mh+SI+4EK-bBd9tnEKo~b~bl1JA1lVUT;2SPA{8tXHrs4SB=!!GiUwZ|NktM z{Oqf*A(j9MAOR$R1dsp{Kmter2_OL^fCP{L5;#}_1JU6Gd5l2(|Ni89DEVRXpOb%; z{9*D6+~9!(kN^@u0!RP}AOR$R1dsp{Kmter2^<80k-m}P!{Iecb-t}js&SrUb4rzC z`*#WN+a)|U+&4UQI4o((hLmCTW>aNz`9^fd(8xVRHqj&N`~N=}w?_|$+YbynZ~8iKV&ta%M4^L%b=}3f?xJ0Hk=vt35^XKy|No1`{xmfBG`!(~ z1dsp{xPb{=zAthl6dfLl9UdNvN1{hw8@tpoi?Q@K2(n*v=$elM>3NQIjZ1cGQcKqfnCVSf*I>3g6sx z8m2)p4O3T`bP6u1lgNZHkrxdLXF_dQ3xHSDl8aYfPYxX(+bMI2Qbf zjcU|e{8|Abi4*G;1_XZiF5%TKUV6)s!I@P+i0C$a<@RnjH`^a)>433}p!YCD3&@_fdVD!Dodh5kvw z<=oN=T;<4Rh4fGeB|!|ifp9;C7qg^TBwe9w%I6iQFlSndz!gfSp?GheWM#QnhW>Fu zfF3Pyw65qvI-g}v(PTtGKvmT==oD~8x-SFWx8suZWYYPevYsAE(@fddl=D>OSU9%p z%I%|U&V-(8)PzYGtX(78%j#LvI)7ZX1i$AWY^U=wC_A;HWeLIcA`H#zkr%UVyr*6| z^J1!D+K%DdGp`H4IB2Xp4NH+~cB*iOr8%0X({zT;(m6U$Ptnsf2Z0>T@idpFxeU!^ z&(bMLmK9C0B*(N!vJ`0vauiq&?9wEgI{VVuL(!vey?y$_S08-)So~M-Tv;9a^fDjr zG0iVO&}*87Ez^887-NBHrqDEl)xPCSGoQ_MnCA3era6DJo931Y-On_qJ@bI3Iaq8R zylL*~bb&d(E~dFBuWK{SWE^ywW{y~3VuFbc=Cbh0!7C51DR_aI=D|+$#887A4{|)n z@gT>89G~CIXcs$;_OXrEm~Z^xFV~N~Gj#rMe)#bpdj5apdGPS|k`yV2X+V`!gL^vLYSIbZ9zxVa_?_ZLytpq=5;h$|-IBE)6NDh1wK7D*gmrc@b zv8gr{SY#?NoM7d@qrc`kY+LRch)s|+lF*sF)7SEyuyi3i-Q$Ec+biy^(sd>@h(l)! zdA>8Sp@NT~yBe(3y>(b;oZJ8hsI@)>{|6jQzw)gU$1*c>MW&=q%xKK<6BXu}nevjj zG+Pw2EGx2;F#gHASUSbbXeE_+sFJ0^O1tW6=NSuD!KPk5U1qAXEZP@rQFg1UET%W$ zA7d$NilrE`(vF?UWW-s>QeLZrx=f~|a$PQ{ve7gxM<^|pimQfe*os4y7gm-}iN)uZ z7MCkxvGfdt+qUq?*m}d1bai~ySW}G}#nhCGjNQ-~YQhEYih2Zus0nAYsZeXwgbx1^ zDfL*Ir6y{y=5!SBTWi#6%3n0!m{7otr>Fg8$_}pobcM?lIIfV+dm>xw%Pm9H^8{dxWApxViXE1jd4B5&093LYtk_+7U8`uu)G>E)9c0XjeF>RRlkZJp~ ziq2s%X+C-mhEvaT1+QbJUOM|^^w{@SAFut+1Alz1{!c|{i$P*H~xJ3xkFXY{|`M7ir)!uJ;xQZe?J%Q>NUA= zzxAK!!hI<&9LI{YnM})t#K5tkT?pn)#4KCc) z8liKo>aDwP4i~QX)7w2=IN#sfVWogB+~QRF6|--@8UKxIn(+rpIOBiWGd_7>OCH?fjNdU@+E8xZ89%7P z%{t@nS^jI7@wYZ?b(*%DXvY6s4{ckUu(og8!WsXI%=mkC!W(_Y56b?te|GY%{{!We{0#sA literal 0 HcmV?d00001 diff --git a/internal/repository/testdata/job.db-shm b/internal/repository/testdata/job.db-shm new file mode 100644 index 0000000000000000000000000000000000000000..fe9ac2845eca6fe6da8a63cd096d9cf9e24ece10 GIT binary patch literal 32768 zcmeIuAr62r3J~xA2ybLe%X9hur>kRx;`PDd1@O@{0$FY#@82dri zMz(H->#PyXvzUXK8JV^*Wlo&vDbdW&%`Pq~%GkzOl9-f}T3Vc11j0O<4H&B#83iY= zVUl7Ln!JHoWb$n$Sw?}$KbfR~q9DE|vy==!o4BqpV}52{N@_(`eo}l=Wqff-Vo^y+ zW^U?aL1u}`<;-&8d~D*%!i>qtia=IOjt9ygVU}RzoqQKa@=gB6Y$w4l$CwIoPOCbG zb9@+?HPv}0&t_F);ca5)oP3m3jgf2e8`g(F@*JCmEO%oCJG;1~BxAETSTE2$C5h<} zl4G(m`vX=EHgS8&$@5tyCM$68iE}m@u(FE_3o|ykgVh5S0}e~4+1SJ_ zg+Z!?fx_{IK!UwThZ(G6hW%y(&abSK5Ad=Xg!*{~hPoo zNAK(XgV?{uem^$b^Xr~Bc2n|Yguk2|>>n8(j(#WVNL5X7ZN(z5JLZ;)GnFD$nVDTE zQY3^L8%&V1A?XTrNwVaIWR3CJ>^QZwJb!v-`8n#T;&W7Ku|kzr78dBiL{qZub+c7Ow zrYuFWO~anM&;QDZ@ zS>{V8iqBHvNKyc5u|&eCF%sb=8J|z~jf@UO>tSUiR;|JNp`En(vEAC#+W6AleIpY? zQ8TOxGO-*-)k)csJ85eRw9Dp&h$PK;@vgp+hliqaSR?6TpzQ9QlzH8-qoO2a{FyuZ zM~)6fFDKf1NZoDFa@RI3UW}(AgiF?3+fgjqcB{dqr#SIoR3T&Il~EiU!Kgmyu`xb+ zN8iZf!_kP>c=kmNth6Y(j^E8h(hed=-tT;=f1rP4bTs<%X|FY8(^ymMt|d9DX>{Bq zTCMKeu8yc(jTnq$B6vaB>@*D!eo4|?g>;#)$$;>%*&8DzB({YHBju?X2Z@u5%fqH2u{2jKw+w5Lc$^H96UBuh7~Z*=^4!dcA_-YpIzdcvX_1DnlDv~ zr;5w87Y6CJg>@kD6Va3}2t0_U4kW%r2M`FN#=zKYC$U5NmgX=}2-+q6_+$NjBggLF zjT&T3)N%fK;BqnEKQc5Fed(gd38LVyw}X(El~*Z1;~yNWqAhjp{k1L8tO&pnk%LGGb=@EjFg{b;v*-j#>XG+?HidM z+O5(g1qW|^u<{rntB)%$#=L4@lY;I6JmMYI_FAa3+O67CK?m7he@H|+OX!%LKSgTM ziJ)bDLt=Zd*s|7v{9eLjqVi^)K)AaA)Wig}VyGLi644cyCiLc@S4%QDgEanmk8E1= zd`NyTVJMPHtXEq+EbMm*$+DzPF^tbIw;x+;nZLFI(gIyf*I~g#sd*={h7~(^3?@+0p`GqPZsOif&q)qvI`8qAQLh)}W&Xs;Elw*$UmhBMtgw zd9mb`+EQ$973eMNJP{(?t{L4H_M{D07Y(!KF9JR3?OcgWW8G=Ms~v=n@yF?_zNqn!$wV^ z-gFC4heAhIunDy3I)Z-^9oV8f3D=~Au*1D3QIhPam&g`Y$#yJLta*iRZaNLvU&}O1 zU18Quv$n|)Kukc3sY^QSX(M}VEd}&IQWx~Vp0{k~Wx|)2I zY&c$lZOtGVfEY{X^ISSVPMxw$w+RtkHpgW>$r)KTUBeLwIO@z|Sun_V=rqcUTvlKQ zx~m8$CCyf-WwLSFa%_R6W=S_YsY!LAV!Zd>{j8PL7HAEPI+JBLV`d@O?Ar6mUhlF9Y2-bV+(L>HJVxPY-OzHf3K^&Qq0R;n=P#w~w+p z6MC*u6DDD>c8q8*t7lB>!U@$9{GNZXozBak?9_^uB?Q-tFf?yOUd*=fo_g`@3#o=_ zJLC%v(C57_0OO#&7durr%hDXp(`hE#?nu9=&=6IS*(_Dt;vgha&Y|>UV z1-7%AmJP{Lq$S8vU^%c$lWgkTi{}o-j=lNTnGarh@U7zmzk2)X>ewe&_-L1De(8a3 z(=2S6=4-(i3rsVGrWvgEZD*SKY%XM)(|eib{H<=9TPAcr)13Cq1DfVwv32mKxvSF! z=JEiik+|tVV zd^%maxSXGy&t9+^nbV9s#nk7{v$MHV+1WG2@_O@0b9&jFJDZYfx@x4(ojvFO{{N?u z>hW6Y;WD{MozW@K@Nb(<&f1Ug>{2zcn zPriE)6+|6K00|%gB!C2v01`j~NB{{S0VIF~kih36aA)jDwEe&!eA64giIbc56M^te z&mFNNi6F-J|9=Ae|C1jj|2_F{$$v`zkN^@u0!RP}AOR$R1dsp{ zKmter2_S+0FM+#aN29W-H#NmkYJDAdy&ZS)j=NaLUC*7dqlvZ_^8f$!#Q!uh_!PY1 zfdr5M61a&8T)D63Xe2f~6hAUNG|&?}`s&!_Mk1OR7>Ew^_YaU`@cD1He&Yo8H+>v@ z&qqilx&L~IK6@kwb zSQh@m3hNeRQE$2hs6(M6E7NtFt|Ry-(NSz?60S)JWz(ohlq5UqCCO1J$#yJLta*iR zZaNLqpqPfKE6loS);1XehzV#hbxGGH(ULYrO94HQ)CE1JA+D=VL)0}U(8e?r+h!aK ze#AyK>Mee)0FlIrbqh3$x@nlABh^9KrOD0}x~Be4b0^$Ej16={6yP%jUSOCpja_rfWC?0Y{x#EDMHd z>5@iyk;}?OS8)~Lq@>vjwG6EyS&l8R)GX;{CpD=qR4iBVg)7kN3#zU<0+-1^vqLe; zZWZoHgut_WdV=LA*r^JaEoAsYddd@70*xwZ69M{!uc!n)aY3~mL1KA6<4KiVorFUF zq~LOHX$7uw~Ek)o8CDTy6H&3#%Tr5NX zxF|r67C2g0bRnJ3vZrY>A|RluY8rG3xFX$`f$kf+Bt4mQeyFUcN76J?_BG`^RXG-p z?YeUND4R2(=NdI(5(aC>i1xC2#ja^G4*wY#Z;X z7tg+sYM8cT`1Z`}0x%95>rTT`q?(;7oMmZ_=IJz@p|f<3&eK!$G|fREM{_*QrD-lh zbJ=rrN|I$oQ!L3bEs`unT7nz}mIJ#q$)?V|ck zTL*8NyEOfwk=VbjbJ3rtKfvB6vxUO9N>;WY&>Fw;EPX`UErkmEs) z2RR<(c#z}sdl~Iw*k~W$c$N9u5B_rf_}fDl{^o}t{h{ao_dEywKfK|A1P+S8RWtrz zBsMY}KXU&=#8LacHI;}C48W`9r}f?YTKo4e$=6nbpS1ALHY^-9g)AidKaQR`v7^f- z>9*KZn+hy46&Oyi^54;4^BlGc(dSSqd>u3;+1+ zN2JtaX_lI(!J5-iz;CTlt0{lcd~-qpe=^*6sGtu{p0~lNUUgYE4W8Z2Tm>dve&RC2 z6A$mPL5kvU+43uvwF*tM%K|e!JvBYOwZIGm@Ky&c$3L(_V?4*Er$UbZ6w7hGyBawD zzUaZZ{^@CdnX-fHKV9K61&%AE^Pb4o`f|$G+hr1T5^FKe-qn*od%It$IgcgH>VBr9x`oz zRuLW+ljfu6VK{X?SMWMk>cw+U#EyS=_3_&8KJdrK>wo)m?aJKiSNPbz{{OMI|Nm}( zxBve;=>Ol2{(tAAS$-Gu>^+*DzSaHzmI>do&pSPu<#_)A?hb|-AI%2K;RAg%>rXvj z%%fTUdi;Or0$-Bb?AyB8*<MCTe|o#83+MZLJFFDYh5O9Ux304j-jpuf4rTYsxx<!!Hw&cMr&iJ9x(uQ*D&iFwU zZq*ro&+=c#jK8&ED{R_sp&9=(J+y6Y!rH!V3upY#Gvn{o32*ipKPda>oAGo0jQ^eA V`^Wd6pZVVLde1%I`q`;B{}1Kp{0RU6 From 2de85b012051ef3ffd06c010611e1607abf1d26f Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 1 Jun 2023 15:40:37 +0200 Subject: [PATCH 8/8] Add composite indexes --- internal/repository/migrations/mysql/02_add-index.down.sql | 5 ++++- internal/repository/migrations/mysql/02_add-index.up.sql | 5 ++++- .../repository/migrations/sqlite3/02_add-index.down.sql | 5 ++++- internal/repository/migrations/sqlite3/02_add-index.up.sql | 7 +++++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/internal/repository/migrations/mysql/02_add-index.down.sql b/internal/repository/migrations/mysql/02_add-index.down.sql index 8129772..1392c45 100644 --- a/internal/repository/migrations/mysql/02_add-index.down.sql +++ b/internal/repository/migrations/mysql/02_add-index.down.sql @@ -2,4 +2,7 @@ DROP INDEX IF EXISTS job_stats; DROP INDEX IF EXISTS job_by_user; DROP INDEX IF EXISTS job_by_starttime; DROP INDEX IF EXISTS job_by_job_id; -DROP INDEX IF EXISTS job_by_state; +DROP INDEX IF EXISTS job_list; +DROP INDEX IF EXISTS job_list_user; +DROP INDEX IF EXISTS job_list_users; +DROP INDEX IF EXISTS job_list_users_start; diff --git a/internal/repository/migrations/mysql/02_add-index.up.sql b/internal/repository/migrations/mysql/02_add-index.up.sql index 7d8d04a..2524bd9 100644 --- a/internal/repository/migrations/mysql/02_add-index.up.sql +++ b/internal/repository/migrations/mysql/02_add-index.up.sql @@ -2,4 +2,7 @@ CREATE INDEX IF NOT EXISTS job_stats ON job (cluster,subcluster,user); CREATE INDEX IF NOT EXISTS job_by_user ON job (user); CREATE INDEX IF NOT EXISTS job_by_starttime ON job (start_time); CREATE INDEX IF NOT EXISTS job_by_job_id ON job (job_id); -CREATE INDEX IF NOT EXISTS job_by_state ON job (job_state); +CREATE INDEX IF NOT EXISTS job_list ON job (cluster, job_state); +CREATE INDEX IF NOT EXISTS job_list_user ON job (user, cluster, job_state); +CREATE INDEX IF NOT EXISTS job_list_users ON job (user, job_state); +CREATE INDEX IF NOT EXISTS job_list_users_start ON job (start_time, user, job_state); diff --git a/internal/repository/migrations/sqlite3/02_add-index.down.sql b/internal/repository/migrations/sqlite3/02_add-index.down.sql index 8129772..1392c45 100644 --- a/internal/repository/migrations/sqlite3/02_add-index.down.sql +++ b/internal/repository/migrations/sqlite3/02_add-index.down.sql @@ -2,4 +2,7 @@ DROP INDEX IF EXISTS job_stats; DROP INDEX IF EXISTS job_by_user; DROP INDEX IF EXISTS job_by_starttime; DROP INDEX IF EXISTS job_by_job_id; -DROP INDEX IF EXISTS job_by_state; +DROP INDEX IF EXISTS job_list; +DROP INDEX IF EXISTS job_list_user; +DROP INDEX IF EXISTS job_list_users; +DROP INDEX IF EXISTS job_list_users_start; diff --git a/internal/repository/migrations/sqlite3/02_add-index.up.sql b/internal/repository/migrations/sqlite3/02_add-index.up.sql index 7d8d04a..db9792d 100644 --- a/internal/repository/migrations/sqlite3/02_add-index.up.sql +++ b/internal/repository/migrations/sqlite3/02_add-index.up.sql @@ -1,5 +1,8 @@ CREATE INDEX IF NOT EXISTS job_stats ON job (cluster,subcluster,user); CREATE INDEX IF NOT EXISTS job_by_user ON job (user); CREATE INDEX IF NOT EXISTS job_by_starttime ON job (start_time); -CREATE INDEX IF NOT EXISTS job_by_job_id ON job (job_id); -CREATE INDEX IF NOT EXISTS job_by_state ON job (job_state); +CREATE INDEX IF NOT EXISTS job_by_job_id ON job (job_id, cluster, start_time); +CREATE INDEX IF NOT EXISTS job_list ON job (cluster, job_state); +CREATE INDEX IF NOT EXISTS job_list_user ON job (user, cluster, job_state); +CREATE INDEX IF NOT EXISTS job_list_users ON job (user, job_state); +CREATE INDEX IF NOT EXISTS job_list_users_start ON job (start_time, user, job_state);