Merge pull request #96 from ClusterCockpit/92_add_shortjobduration_config

Move ShortRunningJobsDuration value to root config

configs/README.md

@@ -24,7 +24,8 @@ It is supported to specify these by means of an `.env` file located in the proje
 * `https-cert-file` and `https-key-file`: Type string. If both those options are not empty, use HTTPS using those certificates.
 * `redirect-http-to`: Type string. If not the empty string and `addr` does not end in ":80", redirect every request incoming at port 80 to that url.
 * `machine-state-dir`: Type string. Where to store MachineState files. TODO: Explain in more detail!
-* `"stop-jobs-exceeding-walltime`: Type int. If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job. Default `0`;
+* `"stop-jobs-exceeding-walltime`: Type int. If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job. Default `0`.
+* `short-running-jobs-duration`: Type int. Do not show running jobs shorter than X seconds. Default `300`.
 * `ldap`: Type object. For LDAP Authentication and user synchronisation. Default `nil`.
    - `url`: Type string.  URL of LDAP directory server.
    - `user_base`: Type string. Base DN of user tree root.
@@ -54,7 +55,6 @@ It is supported to specify these by means of an `.env` file located in the proje
    - `plot_general_colorBackground`: Type bool. Color plot background according to job average threshold limits. Default `true`.
    - `plot_general_colorscheme`: Type string array. Initial color scheme. Default `"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"`.
    - `plot_general_lineWidth`: Type int. Initial linewidth. Default `3`.
-   - `plot_list_hideShortRunningJobs`: Type int. Do not show running jobs shorter than X seconds. Default `300`.
    - `plot_list_jobsPerPage`: Type int. Jobs shown per page in job lists. Default `50`.
    - `plot_list_selectedMetrics`: Type string array. Initial metric plots shown in jobs lists. Default `"cpu_load", "ipc", "mem_used", "flops_any", "mem_bw"`.
    - `plot_view_plotsPerRow`: Type int. Number of plots per row in single job view. Default `3`.

configs/config.json

@@ -35,5 +35,6 @@
         "forceJWTValidationViaDatabase": false,
         "max-age": 0,
         "trustedExternalIssuer": ""
-    }
+    },
+    "short-running-jobs-duration": 300
 }

internal/config/config.go

@@ -25,6 +25,7 @@ var Keys schema.ProgramConfig = schema.ProgramConfig{
 	LdapConfig:                nil,
 	SessionMaxAge:             "168h",
 	StopJobsExceedingWalltime: 0,
+	ShortRunningJobsDuration:  5 * 60,
 	UiDefaults: map[string]interface{}{
 		"analysis_view_histogramMetrics":     []string{"flops_any", "mem_bw", "mem_used"},
 		"analysis_view_scatterPlotMetrics":   [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}},
@@ -34,7 +35,6 @@ var Keys schema.ProgramConfig = schema.ProgramConfig{
 		"plot_general_colorBackground":       true,
 		"plot_general_colorscheme":           []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"},
 		"plot_general_lineWidth":             3,
-		"plot_list_hideShortRunningJobs":     5 * 60,
 		"plot_list_jobsPerPage":              50,
 		"plot_list_selectedMetrics":          []string{"cpu_load", "ipc", "mem_used", "flops_any", "mem_bw"},
 		"plot_view_plotsPerRow":              3,

internal/repository/job.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/99designs/gqlgen/graphql"
 	"github.com/ClusterCockpit/cc-backend/internal/auth"
+	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/internal/metricdata"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
@@ -768,9 +769,6 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error {
 	return nil
 }
 
-// TODO: Move to config
-const ShortJobDuration int = 5 * 60
-
 // GraphQL validation should make sure that no unkown values can be specified.
 var groupBy2column = map[model.Aggregate]string{
 	model.AggregateUser:    "job.user",
@@ -859,7 +857,7 @@ func (r *JobRepository) JobsStatistics(ctx context.Context,
 	}
 
 	if groupBy == nil {
-		query := sq.Select("COUNT(job.id)").From("job").Where("job.duration < ?", ShortJobDuration)
+		query := sq.Select("COUNT(job.id)").From("job").Where("job.duration < ?", config.Keys.ShortRunningJobsDuration)
 		query = SecurityCheck(ctx, query)
 		for _, f := range filter {
 			query = BuildWhereClause(f, query)
@@ -870,7 +868,7 @@ func (r *JobRepository) JobsStatistics(ctx context.Context,
 		}
 	} else {
 		col := groupBy2column[*groupBy]
-		query := sq.Select(col, "COUNT(job.id)").From("job").Where("job.duration < ?", ShortJobDuration)
+		query := sq.Select(col, "COUNT(job.id)").From("job").Where("job.duration < ?", config.Keys.ShortRunningJobsDuration)
 		query = SecurityCheck(ctx, query)
 		for _, f := range filter {
 			query = BuildWhereClause(f, query)

internal/repository/user.go

@@ -11,8 +11,8 @@ import (
 
 	"github.com/ClusterCockpit/cc-backend/internal/auth"
 	"github.com/ClusterCockpit/cc-backend/internal/config"
-	"github.com/ClusterCockpit/cc-backend/pkg/lrucache"
 	"github.com/ClusterCockpit/cc-backend/pkg/log"
+	"github.com/ClusterCockpit/cc-backend/pkg/lrucache"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -63,14 +63,14 @@ func (uCfg *UserCfgRepo) GetUIConfig(user *auth.User) (map[string]interface{}, e
 	}
 
 	data := uCfg.cache.Get(user.Username, func() (interface{}, time.Duration, int) {
-		config := make(map[string]interface{}, len(uCfg.uiDefaults))
+		uiconfig := make(map[string]interface{}, len(uCfg.uiDefaults))
 		for k, v := range uCfg.uiDefaults {
-			config[k] = v
+			uiconfig[k] = v
 		}
 
 		rows, err := uCfg.Lookup.Query(user.Username)
 		if err != nil {
-			log.Warnf("Error while looking up user config for user '%v'", user.Username)
+			log.Warnf("Error while looking up user uiconfig for user '%v'", user.Username)
 			return err, 0, 0
 		}
 
@@ -79,22 +79,25 @@ func (uCfg *UserCfgRepo) GetUIConfig(user *auth.User) (map[string]interface{}, e
 		for rows.Next() {
 			var key, rawval string
 			if err := rows.Scan(&key, &rawval); err != nil {
-				log.Warn("Error while scanning user config values")
+				log.Warn("Error while scanning user uiconfig values")
 				return err, 0, 0
 			}
 
 			var val interface{}
 			if err := json.Unmarshal([]byte(rawval), &val); err != nil {
-				log.Warn("Error while unmarshaling raw user config json")
+				log.Warn("Error while unmarshaling raw user uiconfig json")
 				return err, 0, 0
 			}
 
 			size += len(key)
 			size += len(rawval)
-			config[key] = val
+			uiconfig[key] = val
 		}
 
-		return config, 24 * time.Hour, size
+		// Add global ShortRunningJobsDuration setting as plot_list_hideShortRunningJobs
+		uiconfig["plot_list_hideShortRunningJobs"] = config.Keys.ShortRunningJobsDuration
+
+		return uiconfig, 24 * time.Hour, size
 	})
 	if err, ok := data.(error); ok {
 		log.Error("Error in returned dataset")

internal/routerConfig/routes.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/ClusterCockpit/cc-backend/internal/api"
 	"github.com/ClusterCockpit/cc-backend/internal/auth"
+	"github.com/ClusterCockpit/cc-backend/internal/config"
 	"github.com/ClusterCockpit/cc-backend/internal/graph/model"
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
 	"github.com/ClusterCockpit/cc-backend/pkg/archive"
@@ -72,7 +73,7 @@ func setupHomeRoute(i InfoType, r *http.Request) InfoType {
 	from := time.Now().Add(-24 * time.Hour)
 	recentShortJobs, err := jobRepo.CountGroupedJobs(r.Context(), model.AggregateCluster, []*model.JobFilter{{
 		StartTime: &schema.TimeRange{From: &from, To: nil},
-		Duration:  &schema.IntRange{From: 0, To: repository.ShortJobDuration},
+		Duration:  &schema.IntRange{From: 0, To: config.Keys.ShortRunningJobsDuration},
 	}}, nil, nil)
 	if err != nil {
 		log.Warnf("failed to count jobs: %s", err.Error())

pkg/schema/config.go

@@ -116,6 +116,9 @@ type ProgramConfig struct {
 	// If not zero, automatically mark jobs as stopped running X seconds longer than their walltime.
 	StopJobsExceedingWalltime int `json:"stop-jobs-exceeding-walltime"`
 
+	// Defines time X in seconds in which jobs are considered to be "short" and will be filtered in specific views.
+	ShortRunningJobsDuration int `json:"short-running-jobs-duration"`
+
 	// Array of Clusters
 	Clusters []*ClusterConfig `json:"clusters"`
 }

pkg/schema/schemas/config.schema.json

@@ -76,6 +76,10 @@
             "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.",
             "type": "integer"
         },
+        "short-running-jobs-duration": {
+            "description": "Do not show running jobs shorter than X seconds.",
+            "type": "integer"
+        },
         "": {
             "description": "",
             "type": "string"
@@ -241,10 +245,6 @@
                     "description": "Jobs shown per page in job lists",
                     "type": "integer"
                 },
-                "plot_list_hideShortRunningJobs": {
-                    "description": "Do not show running jobs shorter than X seconds",
-                    "type": "integer"
-                },
                 "plot_view_plotsPerRow": {
                     "description": "Number of plots per row in single job view",
                     "type": "integer"
@@ -342,8 +342,7 @@
                 "job_view_polarPlotMetrics",
                 "job_view_selectedMetrics",
                 "plot_general_colorscheme",
-                "plot_list_selectedMetrics",
+                "plot_list_selectedMetrics"
-                "plot_list_hideShortRunningJobs"
             ]
         }
     },