Move jobs query logic to repository

repository/job.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"database/sql"
 	"errors"
-	"fmt"
 	"strconv"
 
 	"github.com/ClusterCockpit/cc-backend/auth"
@@ -42,8 +41,7 @@ func (r *JobRepository) Find(
 		return nil, err
 	}
 
-	job, err := schema.ScanJob(r.DB.QueryRowx(sqlQuery, args...))
-	return job, err
+	return schema.ScanJob(r.DB.QueryRowx(sqlQuery, args...))
 }
 
 // FindById executes a SQL query to find a specific batch job.
@@ -58,8 +56,7 @@ func (r *JobRepository) FindById(
 		return nil, err
 	}
 
-	job, err := schema.ScanJob(r.DB.QueryRowx(sqlQuery, args...))
-	return job, err
+	return schema.ScanJob(r.DB.QueryRowx(sqlQuery, args...))
 }
 
 // Start inserts a new job in the table, returning the unique job ID.
@@ -96,20 +93,9 @@ func (r *JobRepository) Stop(
 	return
 }
 
-// CountJobs returns the number of jobs for the specified user (if a non-admin user is found in that context) and state.
+// CountJobsPerCluster returns the number of jobs for the specified user (if a non-admin user is found in that context) and state.
 // The counts are grouped by cluster.
-func (r *JobRepository) CountJobs(ctx context.Context, state *schema.JobState) (map[string]int, error) {
-	// q := sq.Select("count(*)").From("job")
-	// if cluster != nil {
-	// 	q = q.Where("job.cluster = ?", cluster)
-	// }
-	// if state != nil {
-	// 	q = q.Where("job.job_state = ?", string(*state))
-	// }
-
-	// err = q.RunWith(r.DB).QueryRow().Scan(&count)
-	// return
-
+func (r *JobRepository) CountJobsPerCluster(ctx context.Context, state *schema.JobState) (map[string]int, error) {
 	q := sq.Select("job.cluster, count(*)").From("job").GroupBy("job.cluster")
 	if state != nil {
 		q = q.Where("job.job_state = ?", string(*state))
@@ -137,13 +123,6 @@ func (r *JobRepository) CountJobs(ctx context.Context, state *schema.JobState) (
 	return counts, nil
 }
 
-// func (r *JobRepository) Query(
-// 	filters []*model.JobFilter,
-// 	page *model.PageRequest,
-// 	order *model.OrderByInput) ([]*schema.Job, int, error) {
-
-// }
-
 func (r *JobRepository) UpdateMonitoringStatus(job int64, monitoringStatus int32) (err error) {
 	stmt := sq.Update("job").
 		Set("monitoring_status", monitoringStatus).
@@ -186,91 +165,6 @@ func (r *JobRepository) Archive(
 	return nil
 }
 
-// Add the tag with id `tagId` to the job with the database id `jobId`.
-func (r *JobRepository) AddTag(jobId int64, tagId int64) error {
-	_, err := r.DB.Exec(`INSERT INTO jobtag (job_id, tag_id) VALUES (?, ?)`, jobId, tagId)
-	return err
-}
-
-// CreateTag creates a new tag with the specified type and name and returns its database id.
-func (r *JobRepository) CreateTag(tagType string, tagName string) (tagId int64, err error) {
-	res, err := r.DB.Exec("INSERT INTO tag (tag_type, tag_name) VALUES ($1, $2)", tagType, tagName)
-	if err != nil {
-		return 0, err
-	}
-
-	return res.LastInsertId()
-}
-
-func (r *JobRepository) GetTags(user *string) (tags []schema.Tag, counts map[string]int, err error) {
-	tags = make([]schema.Tag, 0, 100)
-	xrows, err := r.DB.Queryx("SELECT * FROM tag")
-	if err != nil {
-		return nil, nil, err
-	}
-
-	for xrows.Next() {
-		var t schema.Tag
-		if err := xrows.StructScan(&t); err != nil {
-			return nil, nil, err
-		}
-		tags = append(tags, t)
-	}
-
-	q := sq.Select("t.tag_name, count(jt.tag_id)").
-		From("tag t").
-		LeftJoin("jobtag jt ON t.id = jt.tag_id").
-		GroupBy("t.tag_name")
-	if user != nil {
-		q = q.Where("jt.job_id IN (SELECT id FROM job WHERE job.user = ?)", *user)
-	}
-
-	rows, err := q.RunWith(r.DB).Query()
-	if err != nil {
-		return nil, nil, err
-	}
-
-	counts = make(map[string]int)
-
-	for rows.Next() {
-		var tagName string
-		var count int
-		err = rows.Scan(&tagName, &count)
-		if err != nil {
-			fmt.Println(err)
-		}
-		counts[tagName] = count
-	}
-	err = rows.Err()
-
-	return
-}
-
-// AddTagOrCreate adds the tag with the specified type and name to the job with the database id `jobId`.
-// If such a tag does not yet exist, it is created.
-func (r *JobRepository) AddTagOrCreate(jobId int64, tagType string, tagName string) (tagId int64, err error) {
-	tagId, exists := r.TagId(tagType, tagName)
-	if !exists {
-		tagId, err = r.CreateTag(tagType, tagName)
-		if err != nil {
-			return 0, err
-		}
-	}
-
-	return tagId, r.AddTag(jobId, tagId)
-}
-
-// TagId returns the database id of the tag with the specified type and name.
-func (r *JobRepository) TagId(tagType string, tagName string) (tagId int64, exists bool) {
-	exists = true
-	if err := sq.Select("id").From("tag").
-		Where("tag.tag_type = ?", tagType).Where("tag.tag_name = ?", tagName).
-		RunWith(r.DB).QueryRow().Scan(&tagId); err != nil {
-		exists = false
-	}
-	return
-}
-
 var ErrNotFound = errors.New("no such job or user")
 
 // FindJobOrUser returns a job database ID or a username if a job or user machtes the search term.

repository/job_test.go

@@ -7,8 +7,6 @@ import (
 	"github.com/jmoiron/sqlx"
 
 	"github.com/ClusterCockpit/cc-backend/test"
-	_ "github.com/go-sql-driver/mysql"
-	_ "github.com/mattn/go-sqlite3"
 )
 
 var db *sqlx.DB
@@ -57,7 +55,7 @@ func TestFindById(t *testing.T) {
 func TestGetTags(t *testing.T) {
 	r := setup(t)
 
-	tags, counts, err := r.GetTags(nil)
+	tags, counts, err := r.CountTags(nil)
 	if err != nil {
 		t.Fatal(err)
 	}

repository/query.go

@@ -0,0 +1,212 @@
+package repository
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/ClusterCockpit/cc-backend/auth"
+	"github.com/ClusterCockpit/cc-backend/graph/model"
+	"github.com/ClusterCockpit/cc-backend/log"
+	"github.com/ClusterCockpit/cc-backend/schema"
+	sq "github.com/Masterminds/squirrel"
+)
+
+// QueryJobs returns a list of jobs matching the provided filters. page and order are optional-
+func (r *JobRepository) QueryJobs(
+	ctx context.Context,
+	filters []*model.JobFilter,
+	page *model.PageRequest,
+	order *model.OrderByInput) ([]*schema.Job, error) {
+
+	query := sq.Select(schema.JobColumns...).From("job")
+	query = SecurityCheck(ctx, query)
+
+	if order != nil {
+		field := toSnakeCase(order.Field)
+		if order.Order == model.SortDirectionEnumAsc {
+			query = query.OrderBy(fmt.Sprintf("job.%s ASC", field))
+		} else if order.Order == model.SortDirectionEnumDesc {
+			query = query.OrderBy(fmt.Sprintf("job.%s DESC", field))
+		} else {
+			return nil, errors.New("invalid sorting order")
+		}
+	}
+
+	if page != nil {
+		limit := uint64(page.ItemsPerPage)
+		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
+	} else {
+		query = query.Limit(50)
+	}
+
+	for _, f := range filters {
+		query = BuildWhereClause(f, query)
+	}
+
+	sql, args, err := query.ToSql()
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugf("SQL query: `%s`, args: %#v", sql, args)
+	rows, err := r.DB.Queryx(sql, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	jobs := make([]*schema.Job, 0, 50)
+	for rows.Next() {
+		job, err := schema.ScanJob(rows)
+		if err != nil {
+			return nil, err
+		}
+		jobs = append(jobs, job)
+	}
+
+	return jobs, nil
+}
+
+// CountJobs counts the number of jobs matching the filters.
+func (r *JobRepository) CountJobs(
+	ctx context.Context,
+	filters []*model.JobFilter) (int, error) {
+
+	// count all jobs:
+	query := sq.Select("count(*)").From("job")
+	query = SecurityCheck(ctx, query)
+	for _, f := range filters {
+		query = BuildWhereClause(f, query)
+	}
+	var count int
+	if err := query.RunWith(r.DB).Scan(&count); err != nil {
+		return 0, err
+	}
+
+	return count, nil
+}
+
+func SecurityCheck(ctx context.Context, query sq.SelectBuilder) sq.SelectBuilder {
+	user := auth.GetUser(ctx)
+	if user == nil || user.HasRole(auth.RoleAdmin) {
+		return query
+	}
+
+	return query.Where("job.user = ?", user.Username)
+}
+
+// Build a sq.SelectBuilder out of a schema.JobFilter.
+func BuildWhereClause(filter *model.JobFilter, query sq.SelectBuilder) sq.SelectBuilder {
+	if filter.Tags != nil {
+		query = query.Join("jobtag ON jobtag.job_id = job.id").Where(sq.Eq{"jobtag.tag_id": filter.Tags})
+	}
+	if filter.JobID != nil {
+		query = buildStringCondition("job.job_id", filter.JobID, query)
+	}
+	if filter.ArrayJobID != nil {
+		query = query.Where("job.array_job_id = ?", *filter.ArrayJobID)
+	}
+	if filter.User != nil {
+		query = buildStringCondition("job.user", filter.User, query)
+	}
+	if filter.Project != nil {
+		query = buildStringCondition("job.project", filter.Project, query)
+	}
+	if filter.Cluster != nil {
+		query = buildStringCondition("job.cluster", filter.Cluster, query)
+	}
+	if filter.Partition != nil {
+		query = buildStringCondition("job.partition", filter.Partition, query)
+	}
+	if filter.StartTime != nil {
+		query = buildTimeCondition("job.start_time", filter.StartTime, query)
+	}
+	if filter.Duration != nil {
+		query = buildIntCondition("job.duration", filter.Duration, query)
+	}
+	if filter.State != nil {
+		states := make([]string, len(filter.State))
+		for i, val := range filter.State {
+			states[i] = string(val)
+		}
+
+		query = query.Where(sq.Eq{"job.job_state": states})
+	}
+	if filter.NumNodes != nil {
+		query = buildIntCondition("job.num_nodes", filter.NumNodes, query)
+	}
+	if filter.NumAccelerators != nil {
+		query = buildIntCondition("job.num_acc", filter.NumAccelerators, query)
+	}
+	if filter.NumHWThreads != nil {
+		query = buildIntCondition("job.num_hwthreads", filter.NumHWThreads, query)
+	}
+	if filter.FlopsAnyAvg != nil {
+		query = buildFloatCondition("job.flops_any_avg", filter.FlopsAnyAvg, query)
+	}
+	if filter.MemBwAvg != nil {
+		query = buildFloatCondition("job.mem_bw_avg", filter.MemBwAvg, query)
+	}
+	if filter.LoadAvg != nil {
+		query = buildFloatCondition("job.load_avg", filter.LoadAvg, query)
+	}
+	if filter.MemUsedMax != nil {
+		query = buildFloatCondition("job.mem_used_max", filter.MemUsedMax, query)
+	}
+	return query
+}
+
+func buildIntCondition(field string, cond *model.IntRange, query sq.SelectBuilder) sq.SelectBuilder {
+	return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+}
+
+func buildTimeCondition(field string, cond *model.TimeRange, query sq.SelectBuilder) sq.SelectBuilder {
+	if cond.From != nil && cond.To != nil {
+		return query.Where(field+" BETWEEN ? AND ?", cond.From.Unix(), cond.To.Unix())
+	} else if cond.From != nil {
+		return query.Where("? <= "+field, cond.From.Unix())
+	} else if cond.To != nil {
+		return query.Where(field+" <= ?", cond.To.Unix())
+	} else {
+		return query
+	}
+}
+
+func buildFloatCondition(field string, cond *model.FloatRange, query sq.SelectBuilder) sq.SelectBuilder {
+	return query.Where(field+" BETWEEN ? AND ?", cond.From, cond.To)
+}
+
+func buildStringCondition(field string, cond *model.StringInput, query sq.SelectBuilder) sq.SelectBuilder {
+	if cond.Eq != nil {
+		return query.Where(field+" = ?", *cond.Eq)
+	}
+	if cond.StartsWith != nil {
+		return query.Where(field+" LIKE ?", fmt.Sprint(*cond.StartsWith, "%"))
+	}
+	if cond.EndsWith != nil {
+		return query.Where(field+" LIKE ?", fmt.Sprint("%", *cond.EndsWith))
+	}
+	if cond.Contains != nil {
+		return query.Where(field+" LIKE ?", fmt.Sprint("%", *cond.Contains, "%"))
+	}
+	return query
+}
+
+var matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)")
+var matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])")
+
+func toSnakeCase(str string) string {
+	for _, c := range str {
+		if c == '\'' || c == '\\' {
+			panic("A hacker (probably not)!!!")
+		}
+	}
+
+	str = strings.ReplaceAll(str, "'", "")
+	str = strings.ReplaceAll(str, "\\", "")
+	snake := matchFirstCap.ReplaceAllString(str, "${1}_${2}")
+	snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}")
+	return strings.ToLower(snake)
+}

repository/tags.go

@@ -0,0 +1,93 @@
+package repository
+
+import (
+	"fmt"
+
+	"github.com/ClusterCockpit/cc-backend/schema"
+	sq "github.com/Masterminds/squirrel"
+)
+
+// Add the tag with id `tagId` to the job with the database id `jobId`.
+func (r *JobRepository) AddTag(jobId int64, tagId int64) error {
+	_, err := r.DB.Exec(`INSERT INTO jobtag (job_id, tag_id) VALUES (?, ?)`, jobId, tagId)
+	return err
+}
+
+// CreateTag creates a new tag with the specified type and name and returns its database id.
+func (r *JobRepository) CreateTag(tagType string, tagName string) (tagId int64, err error) {
+	res, err := r.DB.Exec("INSERT INTO tag (tag_type, tag_name) VALUES ($1, $2)", tagType, tagName)
+	if err != nil {
+		return 0, err
+	}
+
+	return res.LastInsertId()
+}
+
+func (r *JobRepository) CountTags(user *string) (tags []schema.Tag, counts map[string]int, err error) {
+	tags = make([]schema.Tag, 0, 100)
+	xrows, err := r.DB.Queryx("SELECT * FROM tag")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for xrows.Next() {
+		var t schema.Tag
+		if err := xrows.StructScan(&t); err != nil {
+			return nil, nil, err
+		}
+		tags = append(tags, t)
+	}
+
+	q := sq.Select("t.tag_name, count(jt.tag_id)").
+		From("tag t").
+		LeftJoin("jobtag jt ON t.id = jt.tag_id").
+		GroupBy("t.tag_name")
+	if user != nil {
+		q = q.Where("jt.job_id IN (SELECT id FROM job WHERE job.user = ?)", *user)
+	}
+
+	rows, err := q.RunWith(r.DB).Query()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	counts = make(map[string]int)
+
+	for rows.Next() {
+		var tagName string
+		var count int
+		err = rows.Scan(&tagName, &count)
+		if err != nil {
+			fmt.Println(err)
+		}
+		counts[tagName] = count
+	}
+	err = rows.Err()
+
+	return
+}
+
+// AddTagOrCreate adds the tag with the specified type and name to the job with the database id `jobId`.
+// If such a tag does not yet exist, it is created.
+func (r *JobRepository) AddTagOrCreate(jobId int64, tagType string, tagName string) (tagId int64, err error) {
+	tagId, exists := r.TagId(tagType, tagName)
+	if !exists {
+		tagId, err = r.CreateTag(tagType, tagName)
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	return tagId, r.AddTag(jobId, tagId)
+}
+
+// TagId returns the database id of the tag with the specified type and name.
+func (r *JobRepository) TagId(tagType string, tagName string) (tagId int64, exists bool) {
+	exists = true
+	if err := sq.Select("id").From("tag").
+		Where("tag.tag_type = ?", tagType).Where("tag.tag_name = ?", tagName).
+		RunWith(r.DB).QueryRow().Scan(&tagId); err != nil {
+		exists = false
+	}
+	return
+}