Refactor for benchmarking

internal/repository/dbConnection.go

@@ -37,8 +37,8 @@ func Connect(driver string, db string) {
 	dbConnOnce.Do(func() {
 		opts := DatabaseOptions{
 			URL:                   db,
-			MaxOpenConnections:    5,
-			MaxIdleConnections:    5,
+			MaxOpenConnections:    4,
+			MaxIdleConnections:    4,
 			ConnectionMaxLifetime: time.Hour,
 			ConnectionMaxIdleTime: time.Hour,
 		}

internal/repository/job.go

@@ -74,7 +74,7 @@ func scanJob(row interface{ Scan(...interface{}) error }) (*schema.Job, error) {
 		&job.ID, &job.JobID, &job.User, &job.Project, &job.Cluster, &job.SubCluster, &job.StartTimeUnix, &job.Partition, &job.ArrayJobId,
 		&job.NumNodes, &job.NumHWThreads, &job.NumAcc, &job.Exclusive, &job.MonitoringStatus, &job.SMT, &job.State,
 		&job.Duration, &job.Walltime, &job.RawResources /*&job.RawMetaData*/); err != nil {
-		log.Warn("Error while scanning rows")
+		log.Warnf("Error while scanning rows: %v", err)
 		return nil, err
 	}
 

internal/repository/job_test.go

@@ -8,21 +8,9 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/ClusterCockpit/cc-backend/pkg/log"
 	_ "github.com/mattn/go-sqlite3"
 )
 
-func setup(t *testing.T) *JobRepository {
-	log.Init("info", true)
-	dbfilepath := "testdata/test.db"
-	err := MigrateDB("sqlite3", dbfilepath)
-	if err != nil {
-		t.Fatal(err)
-	}
-	Connect("sqlite3", dbfilepath)
-	return GetJobRepository()
-}
-
 func TestFind(t *testing.T) {
 	r := setup(t)
 

internal/repository/query.go

@@ -19,19 +19,12 @@ import (
 	sq "github.com/Masterminds/squirrel"
 )
 
-// QueryJobs returns a list of jobs matching the provided filters. page and order are optional-
-func (r *JobRepository) QueryJobs(
-	ctx context.Context,
+func (r *JobRepository) queryJobs(
+	query sq.SelectBuilder,
 	filters []*model.JobFilter,
 	page *model.PageRequest,
 	order *model.OrderByInput) ([]*schema.Job, error) {
 
-	query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job"))
-
-	if qerr != nil {
-		return nil, qerr
-	}
-
 	if order != nil {
 		field := toSnakeCase(order.Field)
 
@@ -81,21 +74,46 @@ func (r *JobRepository) QueryJobs(
 	return jobs, nil
 }
 
-// CountJobs counts the number of jobs matching the filters.
-func (r *JobRepository) CountJobs(
-	ctx context.Context,
-	filters []*model.JobFilter) (int, error) {
+func (r *JobRepository) testQueryJobs(
+	filters []*model.JobFilter,
+	page *model.PageRequest,
+	order *model.OrderByInput) ([]*schema.Job, error) {
 
-	// count all jobs:
-	query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job"))
+	return r.queryJobs(sq.Select(jobColumns...).From("job"),
+		filters, page, order)
+}
+
+// QueryJobs returns a list of jobs matching the provided filters. page and order are optional-
+func (r *JobRepository) QueryJobs(
+	ctx context.Context,
+	filters []*model.JobFilter,
+	page *model.PageRequest,
+	order *model.OrderByInput) ([]*schema.Job, error) {
+
+	query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job"))
 
 	if qerr != nil {
-		return 0, qerr
+		return nil, qerr
 	}
 
+	return r.queryJobs(query,
+		filters, page, order)
+}
+
+func (r *JobRepository) countJobs(query sq.SelectBuilder,
+	filters []*model.JobFilter) (int, error) {
+
 	for _, f := range filters {
 		query = BuildWhereClause(f, query)
 	}
+
+	sql, args, err := query.ToSql()
+	if err != nil {
+		log.Warn("Error while converting query to sql")
+		return 0, nil
+	}
+
+	log.Debugf("SQL query: `%s`, args: %#v", sql, args)
 	var count int
 	if err := query.RunWith(r.DB).Scan(&count); err != nil {
 		return 0, err
@@ -104,6 +122,25 @@ func (r *JobRepository) CountJobs(
 	return count, nil
 }
 
+func (r *JobRepository) testCountJobs(
+	filters []*model.JobFilter) (int, error) {
+
+	return r.countJobs(sq.Select("count(*)").From("job"), filters)
+}
+
+func (r *JobRepository) CountJobs(
+	ctx context.Context,
+	filters []*model.JobFilter) (int, error) {
+
+	query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job"))
+
+	if qerr != nil {
+		return 0, qerr
+	}
+
+	return r.countJobs(query, filters)
+}
+
 func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (queryOut sq.SelectBuilder, err error) {
 	user := auth.GetUser(ctx)
 	if user == nil || user.HasAnyRole([]auth.Role{auth.RoleAdmin, auth.RoleSupport, auth.RoleApi}) { // Admin & Co. : All jobs

internal/repository/repository_bench.go

@@ -0,0 +1,119 @@
+// Copyright (C) 2022 NHR@FAU, University Erlangen-Nuremberg.
+// All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+package repository
+
+import (
+	"testing"
+
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	_ "github.com/mattn/go-sqlite3"
+)
+
+func TestPragma(t *testing.T) {
+	t.Run("sets up a new DB", func(t *testing.T) {
+		db := setup(t)
+
+		for _, pragma := range []string{"synchronous", "journal_mode", "busy_timeout", "auto_vacuum", "foreign_keys"} {
+			t.Log("PRAGMA", pragma, getPragma(db, pragma))
+		}
+	})
+}
+
+func getPragma(db *JobRepository, name string) string {
+	var s string
+	if err := db.DB.QueryRow(`PRAGMA ` + name).Scan(&s); err != nil {
+		panic(err)
+	}
+	return s
+}
+
+func BenchmarkSelect1(b *testing.B) {
+	db := setup(b)
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			_, err := db.DB.Exec(`select 1`)
+			noErr(b, err)
+		}
+	})
+}
+
+func BenchmarkDB_FindJobById(b *testing.B) {
+	var jobId int64 = 1677322
+
+	b.Run("FindJobById", func(b *testing.B) {
+		db := setup(b)
+
+		b.ResetTimer()
+
+		b.RunParallel(func(pb *testing.PB) {
+			for pb.Next() {
+				_, err := db.FindById(jobId)
+				noErr(b, err)
+			}
+		})
+	})
+}
+
+func BenchmarkDB_FindJob(b *testing.B) {
+	var jobId int64 = 107266
+	var startTime int64 = 1657557241
+	var cluster = "fritz"
+
+	b.Run("FindJob", func(b *testing.B) {
+		db := setup(b)
+
+		b.ResetTimer()
+
+		b.RunParallel(func(pb *testing.PB) {
+			for pb.Next() {
+				_, err := db.Find(&jobId, &cluster, &startTime)
+				noErr(b, err)
+			}
+		})
+	})
+}
+
+func BenchmarkDB_CountJobs(b *testing.B) {
+	filter := &model.JobFilter{}
+	filter.State = append(filter.State, "running")
+	cluster := "fritz"
+	filter.Cluster = &model.StringInput{Eq: &cluster}
+
+	b.Run("CountJobs", func(b *testing.B) {
+		db := setup(b)
+
+		b.ResetTimer()
+
+		b.RunParallel(func(pb *testing.PB) {
+			for pb.Next() {
+				_, err := db.testCountJobs([]*model.JobFilter{filter})
+				noErr(b, err)
+			}
+		})
+	})
+}
+
+func setup(tb testing.TB) *JobRepository {
+	tb.Helper()
+	log.Init("warn", true)
+	dbfile := "testdata/job.db"
+	err := MigrateDB("sqlite3", dbfile)
+	noErr(tb, err)
+
+	Connect("sqlite3", dbfile)
+	return GetJobRepository()
+}
+
+func noErr(tb testing.TB, err error) {
+	tb.Helper()
+
+	if err != nil {
+		tb.Fatal("Error is not nil:", err)
+	}
+}