diff --git a/internal/repository/dbConnection.go b/internal/repository/dbConnection.go index 26154d5..da3d40e 100644 --- a/internal/repository/dbConnection.go +++ b/internal/repository/dbConnection.go @@ -37,8 +37,8 @@ func Connect(driver string, db string) { dbConnOnce.Do(func() { opts := DatabaseOptions{ URL: db, - MaxOpenConnections: 5, - MaxIdleConnections: 5, + MaxOpenConnections: 4, + MaxIdleConnections: 4, ConnectionMaxLifetime: time.Hour, ConnectionMaxIdleTime: time.Hour, } diff --git a/internal/repository/job.go b/internal/repository/job.go index 4eee0d3..c5df610 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -74,7 +74,7 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) { &job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId, &job.NumNodes, &job.NumHWThreads, &job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State, &job.Duration, &job.Walltime, &job.RawResources /*&job.RawMetaData*/); err != nil { - log.Warn("Error while scanning rows") + log.Warnf("Error while scanning rows: %v", err) return nil, err } diff --git a/internal/repository/job_test.go b/internal/repository/job_test.go index d74dad5..bf491f4 100644 --- a/internal/repository/job_test.go +++ b/internal/repository/job_test.go @@ -8,21 +8,9 @@ import ( "fmt" "testing" - "github.com/ClusterCockpit/cc-backend/pkg/log" _ "github.com/mattn/go-sqlite3" ) -func setup(t *testing.T) *JobRepository { - log.Init("info", true) - dbfilepath := "testdata/test.db" - err := MigrateDB("sqlite3", dbfilepath) - if err != nil { - t.Fatal(err) - } - Connect("sqlite3", dbfilepath) - return GetJobRepository() -} - func TestFind(t *testing.T) { r := setup(t) diff --git a/internal/repository/query.go b/internal/repository/query.go index 60354c1..5513d51 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -19,19 +19,12 @@ import ( sq "github.com/Masterminds/squirrel" ) -// QueryJobs returns a list of jobs matching the provided filters. page and order are optional- -func (r *JobRepository) QueryJobs( - ctx context.Context, +func (r *JobRepository) queryJobs( + query sq.SelectBuilder, filters []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) ([]*schema.Job, error) { - query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job")) - - if qerr != nil { - return nil, qerr - } - if order != nil { field := toSnakeCase(order.Field) @@ -81,21 +74,46 @@ func (r *JobRepository) QueryJobs( return jobs, nil } -// CountJobs counts the number of jobs matching the filters. -func (r *JobRepository) CountJobs( - ctx context.Context, - filters []*model.JobFilter) (int, error) { +func (r *JobRepository) testQueryJobs( + filters []*model.JobFilter, + page *model.PageRequest, + order *model.OrderByInput) ([]*schema.Job, error) { - // count all jobs: - query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job")) + return r.queryJobs(sq.Select(jobColumns...).From("job"), + filters, page, order) +} + +// QueryJobs returns a list of jobs matching the provided filters. page and order are optional- +func (r *JobRepository) QueryJobs( + ctx context.Context, + filters []*model.JobFilter, + page *model.PageRequest, + order *model.OrderByInput) ([]*schema.Job, error) { + + query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job")) if qerr != nil { - return 0, qerr + return nil, qerr } + return r.queryJobs(query, + filters, page, order) +} + +func (r *JobRepository) countJobs(query sq.SelectBuilder, + filters []*model.JobFilter) (int, error) { + for _, f := range filters { query = BuildWhereClause(f, query) } + + sql, args, err := query.ToSql() + if err != nil { + log.Warn("Error while converting query to sql") + return 0, nil + } + + log.Debugf("SQL query: `%s`, args: %#v", sql, args) var count int if err := query.RunWith(r.DB).Scan(&count); err != nil { return 0, err @@ -104,6 +122,25 @@ func (r *JobRepository) CountJobs( return count, nil } +func (r *JobRepository) testCountJobs( + filters []*model.JobFilter) (int, error) { + + return r.countJobs(sq.Select("count(*)").From("job"), filters) +} + +func (r *JobRepository) CountJobs( + ctx context.Context, + filters []*model.JobFilter) (int, error) { + + query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job")) + + if qerr != nil { + return 0, qerr + } + + return r.countJobs(query, filters) +} + func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (queryOut sq.SelectBuilder, err error) { user := auth.GetUser(ctx) if user == nil || user.HasAnyRole([]auth.Role{auth.RoleAdmin, auth.RoleSupport, auth.RoleApi}) { // Admin & Co. : All jobs diff --git a/internal/repository/repository_bench.go b/internal/repository/repository_bench.go new file mode 100644 index 0000000..598d379 --- /dev/null +++ b/internal/repository/repository_bench.go @@ -0,0 +1,119 @@ +// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg. +// All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +package repository + +import ( + "testing" + + "github.com/ClusterCockpit/cc-backend/internal/graph/model" + "github.com/ClusterCockpit/cc-backend/pkg/log" + _ "github.com/mattn/go-sqlite3" +) + +func TestPragma(t *testing.T) { + t.Run("sets up a new DB", func(t *testing.T) { + db := setup(t) + + for _, pragma := range []string{"synchronous", "journal_mode", "busy_timeout", "auto_vacuum", "foreign_keys"} { + t.Log("PRAGMA", pragma, getPragma(db, pragma)) + } + }) +} + +func getPragma(db *JobRepository, name string) string { + var s string + if err := db.DB.QueryRow(`PRAGMA ` + name).Scan(&s); err != nil { + panic(err) + } + return s +} + +func BenchmarkSelect1(b *testing.B) { + db := setup(b) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := db.DB.Exec(`select 1`) + noErr(b, err) + } + }) +} + +func BenchmarkDB_FindJobById(b *testing.B) { + var jobId int64 = 1677322 + + b.Run("FindJobById", func(b *testing.B) { + db := setup(b) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := db.FindById(jobId) + noErr(b, err) + } + }) + }) +} + +func BenchmarkDB_FindJob(b *testing.B) { + var jobId int64 = 107266 + var startTime int64 = 1657557241 + var cluster = "fritz" + + b.Run("FindJob", func(b *testing.B) { + db := setup(b) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := db.Find(&jobId, &cluster, &startTime) + noErr(b, err) + } + }) + }) +} + +func BenchmarkDB_CountJobs(b *testing.B) { + filter := &model.JobFilter{} + filter.State = append(filter.State, "running") + cluster := "fritz" + filter.Cluster = &model.StringInput{Eq: &cluster} + + b.Run("CountJobs", func(b *testing.B) { + db := setup(b) + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := db.testCountJobs([]*model.JobFilter{filter}) + noErr(b, err) + } + }) + }) +} + +func setup(tb testing.TB) *JobRepository { + tb.Helper() + log.Init("warn", true) + dbfile := "testdata/job.db" + err := MigrateDB("sqlite3", dbfile) + noErr(tb, err) + + Connect("sqlite3", dbfile) + return GetJobRepository() +} + +func noErr(tb testing.TB, err error) { + tb.Helper() + + if err != nil { + tb.Fatal("Error is not nil:", err) + } +}