Merge pull request #205 from ClusterCockpit/166_add_scopes_analysis

166 add scopes analysis

internal/repository/job.go

@@ -455,69 +455,6 @@ func (r *JobRepository) DeleteJobById(id int64) error {
 	return err
 }
 
-// TODO: Use node hours instead: SELECT job.user, sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN CAST(strftime('%s', 'now') AS INTEGER) - job.start_time ELSE job.duration END)) as x FROM job GROUP BY user ORDER BY x DESC;
-func (r *JobRepository) CountGroupedJobs(
-	ctx context.Context,
-	aggreg model.Aggregate,
-	filters []*model.JobFilter,
-	weight *model.Weights,
-	limit *int) (map[string]int, error) {
-
-	start := time.Now()
-	if !aggreg.IsValid() {
-		return nil, errors.New("invalid aggregate")
-	}
-
-	runner := (sq.BaseRunner)(r.stmtCache)
-	count := "count(*) as count"
-	if weight != nil {
-		switch *weight {
-		case model.WeightsNodeCount:
-			count = "sum(job.num_nodes) as count"
-		case model.WeightsNodeHours:
-			now := time.Now().Unix()
-			count = fmt.Sprintf(`sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) as count`, now)
-			runner = r.DB
-		default:
-			log.Debugf("CountGroupedJobs() Weight %v unknown.", *weight)
-		}
-	}
-
-	q, qerr := SecurityCheck(ctx, sq.Select("job."+string(aggreg), count).From("job").GroupBy("job."+string(aggreg)).OrderBy("count DESC"))
-
-	if qerr != nil {
-		return nil, qerr
-	}
-
-	for _, f := range filters {
-		q = BuildWhereClause(f, q)
-	}
-	if limit != nil {
-		q = q.Limit(uint64(*limit))
-	}
-
-	counts := map[string]int{}
-	rows, err := q.RunWith(runner).Query()
-	if err != nil {
-		log.Error("Error while running query")
-		return nil, err
-	}
-
-	for rows.Next() {
-		var group string
-		var count int
-		if err := rows.Scan(&group, &count); err != nil {
-			log.Warn("Error while scanning rows")
-			return nil, err
-		}
-
-		counts[group] = count
-	}
-
-	log.Debugf("Timer CountGroupedJobs %s", time.Since(start))
-	return counts, nil
-}
-
 func (r *JobRepository) UpdateMonitoringStatus(job int64, monitoringStatus int32) (err error) {
 	stmt := sq.Update("job").
 		Set("monitoring_status", monitoringStatus).

internal/repository/query.go

@@ -18,13 +18,17 @@ import (
 	sq "github.com/Masterminds/squirrel"
 )
 
-// SecurityCheck-less, private: Returns a list of jobs matching the provided filters. page and order are optional-
-func (r *JobRepository) queryJobs(
-	query sq.SelectBuilder,
+func (r *JobRepository) QueryJobs(
+	ctx context.Context,
 	filters []*model.JobFilter,
 	page *model.PageRequest,
 	order *model.OrderByInput) ([]*schema.Job, error) {
 
+	query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job"))
+	if qerr != nil {
+		return nil, qerr
+	}
+
 	if order != nil {
 		field := toSnakeCase(order.Field)
 
@@ -67,34 +71,15 @@ func (r *JobRepository) queryJobs(
 	return jobs, nil
 }
 
-// testFunction for queryJobs
-func (r *JobRepository) testQueryJobs(
-	filters []*model.JobFilter,
-	page *model.PageRequest,
-	order *model.OrderByInput) ([]*schema.Job, error) {
-
-	return r.queryJobs(sq.Select(jobColumns...).From("job"), filters, page, order)
-}
-
-// Public function with added securityCheck, calls private queryJobs function above
-func (r *JobRepository) QueryJobs(
+func (r *JobRepository) CountJobs(
 	ctx context.Context,
-	filters []*model.JobFilter,
-	page *model.PageRequest,
-	order *model.OrderByInput) ([]*schema.Job, error) {
-
-	query, qerr := SecurityCheck(ctx, sq.Select(jobColumns...).From("job"))
-	if qerr != nil {
-		return nil, qerr
-	}
-
-	return r.queryJobs(query, filters, page, order)
-}
-
-// SecurityCheck-less, private: Returns the number of jobs matching the filters
-func (r *JobRepository) countJobs(query sq.SelectBuilder,
 	filters []*model.JobFilter) (int, error) {
 
+	query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job"))
+	if qerr != nil {
+		return 0, qerr
+	}
+
 	for _, f := range filters {
 		query = BuildWhereClause(f, query)
 	}
@@ -107,27 +92,6 @@ func (r *JobRepository) countJobs(query sq.SelectBuilder,
 	return count, nil
 }
 
-// testFunction for countJobs
-func (r *JobRepository) testCountJobs(
-	filters []*model.JobFilter) (int, error) {
-
-	return r.countJobs(sq.Select("count(*)").From("job"), filters)
-}
-
-// Public function with added securityCheck, calls private countJobs function above
-func (r *JobRepository) CountJobs(
-	ctx context.Context,
-	filters []*model.JobFilter) (int, error) {
-
-	query, qerr := SecurityCheck(ctx, sq.Select("count(*)").From("job"))
-
-	if qerr != nil {
-		return 0, qerr
-	}
-
-	return r.countJobs(query, filters)
-}
-
 func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilder, error) {
 	user := GetUserFromContext(ctx)
 	if user == nil {

internal/repository/repository_test.go

@@ -5,10 +5,12 @@
 package repository
 
 import (
+	"context"
 	"testing"
 
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 	_ "github.com/mattn/go-sqlite3"
 )
 
@@ -94,7 +96,7 @@ func BenchmarkDB_CountJobs(b *testing.B) {
 
 		b.RunParallel(func(pb *testing.PB) {
 			for pb.Next() {
-				_, err := db.testCountJobs([]*model.JobFilter{filter})
+				_, err := db.CountJobs(getContext(b), []*model.JobFilter{filter})
 				noErr(b, err)
 			}
 		})
@@ -118,20 +120,37 @@ func BenchmarkDB_QueryJobs(b *testing.B) {
 
 		b.RunParallel(func(pb *testing.PB) {
 			for pb.Next() {
-				_, err := db.testQueryJobs([]*model.JobFilter{filter}, page, order)
+				_, err := db.QueryJobs(getContext(b), []*model.JobFilter{filter}, page, order)
 				noErr(b, err)
 			}
 		})
 	})
 }
 
+func getContext(tb testing.TB) context.Context {
+	tb.Helper()
+
+	var roles []string
+	roles = append(roles, schema.GetRoleString(schema.RoleAdmin))
+	projects := make([]string, 0)
+
+	user := &schema.User{
+		Username:   "demo",
+		Name:       "The man",
+		Roles:      roles,
+		Projects:   projects,
+		AuthSource: schema.AuthViaLDAP,
+	}
+	ctx := context.Background()
+	return context.WithValue(ctx, ContextUserKey, user)
+}
+
 func setup(tb testing.TB) *JobRepository {
 	tb.Helper()
 	log.Init("warn", true)
 	dbfile := "testdata/job.db"
 	err := MigrateDB("sqlite3", dbfile)
 	noErr(tb, err)
-
 	Connect("sqlite3", dbfile)
 	return GetJobRepository()
 }

internal/repository/stats.go

@@ -23,6 +23,17 @@ var groupBy2column = map[model.Aggregate]string{
 	model.AggregateCluster: "job.cluster",
 }
 
+var sortBy2column = map[model.SortByAggregate]string{
+	model.SortByAggregateTotaljobs:      "totalJobs",
+	model.SortByAggregateTotalwalltime:  "totalWalltime",
+	model.SortByAggregateTotalnodes:     "totalNodes",
+	model.SortByAggregateTotalnodehours: "totalNodeHours",
+	model.SortByAggregateTotalcores:     "totalCores",
+	model.SortByAggregateTotalcorehours: "totalCoreHours",
+	model.SortByAggregateTotalaccs:      "totalAccs",
+	model.SortByAggregateTotalacchours:  "totalAccHours",
+}
+
 func (r *JobRepository) buildCountQuery(
 	filter []*model.JobFilter,
 	kind string,
@@ -60,19 +71,26 @@ func (r *JobRepository) buildStatsQuery(
 	castType := r.getCastType()
 
 	if col != "" {
-		// Scan columns: id, totalJobs, totalWalltime, totalNodeHours, totalCoreHours, totalAccHours
-		query = sq.Select(col, "COUNT(job.id)",
-			fmt.Sprintf("CAST(ROUND(SUM(job.duration) / 3600) as %s)", castType),
-			fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes) / 3600) as %s)", castType),
-			fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_hwthreads) / 3600) as %s)", castType),
-			fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_acc) / 3600) as %s)", castType),
+		// Scan columns: id, totalJobs, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours
+		query = sq.Select(col, "COUNT(job.id) as totalJobs",
+			fmt.Sprintf("CAST(ROUND(SUM(job.duration) / 3600) as %s) as totalWalltime", castType),
+			fmt.Sprintf("CAST(SUM(job.num_nodes) as %s) as totalNodes", castType),
+			fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes) / 3600) as %s) as totalNodeHours", castType),
+			fmt.Sprintf("CAST(SUM(job.num_hwthreads) as %s) as totalCores", castType),
+			fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_hwthreads) / 3600) as %s) as totalCoreHours", castType),
+			fmt.Sprintf("CAST(SUM(job.num_acc) as %s) as totalAccs", castType),
+			fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_acc) / 3600) as %s) as totalAccHours", castType),
 		).From("job").GroupBy(col)
+
 	} else {
-		// Scan columns: totalJobs, totalWalltime, totalNodeHours, totalCoreHours, totalAccHours
+		// Scan columns: totalJobs, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours
 		query = sq.Select("COUNT(job.id)",
 			fmt.Sprintf("CAST(ROUND(SUM(job.duration) / 3600) as %s)", castType),
+			fmt.Sprintf("CAST(SUM(job.num_nodes) as %s)", castType),
 			fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_nodes) / 3600) as %s)", castType),
+			fmt.Sprintf("CAST(SUM(job.num_hwthreads) as %s)", castType),
 			fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_hwthreads) / 3600) as %s)", castType),
+			fmt.Sprintf("CAST(SUM(job.num_acc) as %s)", castType),
 			fmt.Sprintf("CAST(ROUND(SUM(job.duration * job.num_acc) / 3600) as %s)", castType),
 		).From("job")
 	}
@@ -112,16 +130,28 @@ func (r *JobRepository) getCastType() string {
 func (r *JobRepository) JobsStatsGrouped(
 	ctx context.Context,
 	filter []*model.JobFilter,
+	page *model.PageRequest,
+	sortBy *model.SortByAggregate,
 	groupBy *model.Aggregate) ([]*model.JobsStatistics, error) {
 
 	start := time.Now()
 	col := groupBy2column[*groupBy]
 	query := r.buildStatsQuery(filter, col)
+
 	query, err := SecurityCheck(ctx, query)
 	if err != nil {
 		return nil, err
 	}
 
+	if sortBy != nil {
+		sortBy := sortBy2column[*sortBy]
+		query = query.OrderBy(fmt.Sprintf("%s DESC", sortBy))
+	}
+	if page != nil && page.ItemsPerPage != -1 {
+		limit := uint64(page.ItemsPerPage)
+		query = query.Offset((uint64(page.Page) - 1) * limit).Limit(limit)
+	}
+
 	rows, err := query.RunWith(r.DB).Query()
 	if err != nil {
 		log.Warn("Error while querying DB for job statistics")
@@ -132,15 +162,36 @@ func (r *JobRepository) JobsStatsGrouped(
 
 	for rows.Next() {
 		var id sql.NullString
-		var jobs, walltime, nodeHours, coreHours, accHours sql.NullInt64
-		if err := rows.Scan(&id, &jobs, &walltime, &nodeHours, &coreHours, &accHours); err != nil {
+		var jobs, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64
+		if err := rows.Scan(&id, &jobs, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours); err != nil {
 			log.Warn("Error while scanning rows")
 			return nil, err
 		}
 
 		if id.Valid {
-			var totalCoreHours, totalAccHours int
+			var totalJobs, totalWalltime, totalNodes, totalNodeHours, totalCores, totalCoreHours, totalAccs, totalAccHours int
 
+			if jobs.Valid {
+				totalJobs = int(jobs.Int64)
+			}
+
+			if walltime.Valid {
+				totalWalltime = int(walltime.Int64)
+			}
+
+			if nodes.Valid {
+				totalNodes = int(nodes.Int64)
+			}
+			if cores.Valid {
+				totalCores = int(cores.Int64)
+			}
+			if accs.Valid {
+				totalAccs = int(accs.Int64)
+			}
+
+			if nodeHours.Valid {
+				totalNodeHours = int(nodeHours.Int64)
+			}
 			if coreHours.Valid {
 				totalCoreHours = int(coreHours.Int64)
 			}
@@ -154,9 +205,13 @@ func (r *JobRepository) JobsStatsGrouped(
 					&model.JobsStatistics{
 						ID:             id.String,
 						Name:           name,
-						TotalJobs:      int(jobs.Int64),
-						TotalWalltime:  int(walltime.Int64),
+						TotalJobs:      totalJobs,
+						TotalWalltime:  totalWalltime,
+						TotalNodes:     totalNodes,
+						TotalNodeHours: totalNodeHours,
+						TotalCores:     totalCores,
 						TotalCoreHours: totalCoreHours,
+						TotalAccs:      totalAccs,
 						TotalAccHours:  totalAccHours})
 			} else {
 				stats = append(stats,
@@ -164,7 +219,11 @@ func (r *JobRepository) JobsStatsGrouped(
 						ID:             id.String,
 						TotalJobs:      int(jobs.Int64),
 						TotalWalltime:  int(walltime.Int64),
+						TotalNodes:     totalNodes,
+						TotalNodeHours: totalNodeHours,
+						TotalCores:     totalCores,
 						TotalCoreHours: totalCoreHours,
+						TotalAccs:      totalAccs,
 						TotalAccHours:  totalAccHours})
 			}
 		}
@@ -188,15 +247,18 @@ func (r *JobRepository) JobsStats(
 	row := query.RunWith(r.DB).QueryRow()
 	stats := make([]*model.JobsStatistics, 0, 1)
 
-	var jobs, walltime, nodeHours, coreHours, accHours sql.NullInt64
-	if err := row.Scan(&jobs, &walltime, &nodeHours, &coreHours, &accHours); err != nil {
+	var jobs, walltime, nodes, nodeHours, cores, coreHours, accs, accHours sql.NullInt64
+	if err := row.Scan(&jobs, &walltime, &nodes, &nodeHours, &cores, &coreHours, &accs, &accHours); err != nil {
 		log.Warn("Error while scanning rows")
 		return nil, err
 	}
 
 	if jobs.Valid {
-		var totalCoreHours, totalAccHours int
+		var totalNodeHours, totalCoreHours, totalAccHours int
 
+		if nodeHours.Valid {
+			totalNodeHours = int(nodeHours.Int64)
+		}
 		if coreHours.Valid {
 			totalCoreHours = int(coreHours.Int64)
 		}
@@ -207,6 +269,7 @@ func (r *JobRepository) JobsStats(
 			&model.JobsStatistics{
 				TotalJobs:      int(jobs.Int64),
 				TotalWalltime:  int(walltime.Int64),
+				TotalNodeHours: totalNodeHours,
 				TotalCoreHours: totalCoreHours,
 				TotalAccHours:  totalAccHours})
 	}
@@ -321,7 +384,7 @@ func (r *JobRepository) AddJobCount(
 		return nil, err
 	}
 
-	counts := make(map[string]int)
+	var count int
 
 	for rows.Next() {
 		var cnt sql.NullInt64
@@ -329,20 +392,22 @@ func (r *JobRepository) AddJobCount(
 			log.Warn("Error while scanning rows")
 			return nil, err
 		}
+
+		count = int(cnt.Int64)
 	}
 
 	switch kind {
 	case "running":
 		for _, s := range stats {
-			s.RunningJobs = counts[s.ID]
+			s.RunningJobs = count
 		}
 	case "short":
 		for _, s := range stats {
-			s.ShortJobs = counts[s.ID]
+			s.ShortJobs = count
 		}
 	}
 
-	log.Debugf("Timer JobJobCount %s", time.Since(start))
+	log.Debugf("Timer AddJobCount %s", time.Since(start))
 	return stats, nil
 }
 
@@ -367,6 +432,18 @@ func (r *JobRepository) AddHistograms(
 		return nil, err
 	}
 
+	stat.HistNumCores, err = r.jobsStatisticsHistogram(ctx, "job.num_hwthreads as value", filter)
+	if err != nil {
+		log.Warn("Error while loading job statistics histogram: num hwthreads")
+		return nil, err
+	}
+
+	stat.HistNumAccs, err = r.jobsStatisticsHistogram(ctx, "job.num_acc as value", filter)
+	if err != nil {
+		log.Warn("Error while loading job statistics histogram: num acc")
+		return nil, err
+	}
+
 	log.Debugf("Timer AddHistograms %s", time.Since(start))
 	return stat, nil
 }

internal/repository/stats_test.go

@@ -7,6 +7,8 @@ package repository
 import (
 	"fmt"
 	"testing"
+
+	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 )
 
 func TestBuildJobStatsQuery(t *testing.T) {
@@ -19,3 +21,15 @@ func TestBuildJobStatsQuery(t *testing.T) {
 	fmt.Printf("SQL: %s\n", sql)
 
 }
+
+func TestJobStats(t *testing.T) {
+	r := setup(t)
+
+	filter := &model.JobFilter{}
+	stats, err := r.JobsStats(getContext(t), []*model.JobFilter{filter})
+	noErr(t, err)
+
+	if stats[0].TotalJobs != 6 {
+		t.Fatalf("Want 98, Got %d", stats[0].TotalJobs)
+	}
+}