From 4f1f034702a9294994d479c27c3d536261ac9eeb Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 1 Mar 2023 16:34:59 +0100 Subject: [PATCH] Move ShortRunningJobsDuration value to root config --- configs/README.md | 4 ++-- configs/config.json | 3 ++- internal/config/config.go | 2 +- internal/repository/job.go | 8 +++----- internal/repository/user.go | 19 +++++++++++-------- internal/routerConfig/routes.go | 3 ++- pkg/schema/config.go | 3 +++ pkg/schema/schemas/config.schema.json | 11 +++++------ 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/configs/README.md b/configs/README.md index 10c268f..a4edd0f 100644 --- a/configs/README.md +++ b/configs/README.md @@ -24,7 +24,8 @@ It is supported to specify these by means of an `.env` file located in the proje * `https-cert-file` and `https-key-file`: Type string. If both those options are not empty, use HTTPS using those certificates. * `redirect-http-to`: Type string. If not the empty string and `addr` does not end in ":80", redirect every request incoming at port 80 to that url. * `machine-state-dir`: Type string. Where to store MachineState files. TODO: Explain in more detail! -* `"stop-jobs-exceeding-walltime`: Type int. If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job. Default `0`; +* `"stop-jobs-exceeding-walltime`: Type int. If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job. Default `0`. +* `short-running-jobs-duration`: Type int. Do not show running jobs shorter than X seconds. Default `300`. * `ldap`: Type object. For LDAP Authentication and user synchronisation. Default `nil`. - `url`: Type string. URL of LDAP directory server. - `user_base`: Type string. Base DN of user tree root. @@ -54,7 +55,6 @@ It is supported to specify these by means of an `.env` file located in the proje - `plot_general_colorBackground`: Type bool. Color plot background according to job average threshold limits. Default `true`. - `plot_general_colorscheme`: Type string array. Initial color scheme. Default `"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"`. - `plot_general_lineWidth`: Type int. Initial linewidth. Default `3`. - - `plot_list_hideShortRunningJobs`: Type int. Do not show running jobs shorter than X seconds. Default `300`. - `plot_list_jobsPerPage`: Type int. Jobs shown per page in job lists. Default `50`. - `plot_list_selectedMetrics`: Type string array. Initial metric plots shown in jobs lists. Default `"cpu_load", "ipc", "mem_used", "flops_any", "mem_bw"`. - `plot_view_plotsPerRow`: Type int. Number of plots per row in single job view. Default `3`. diff --git a/configs/config.json b/configs/config.json index 6c838c5..e8e4957 100644 --- a/configs/config.json +++ b/configs/config.json @@ -35,5 +35,6 @@ "forceJWTValidationViaDatabase": false, "max-age": 0, "trustedExternalIssuer": "" - } + }, + "short-running-jobs-duration": 300 } diff --git a/internal/config/config.go b/internal/config/config.go index b8afc9a..d50fa4f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -25,6 +25,7 @@ var Keys schema.ProgramConfig = schema.ProgramConfig{ LdapConfig: nil, SessionMaxAge: "168h", StopJobsExceedingWalltime: 0, + ShortRunningJobsDuration: 5 * 60, UiDefaults: map[string]interface{}{ "analysis_view_histogramMetrics": []string{"flops_any", "mem_bw", "mem_used"}, "analysis_view_scatterPlotMetrics": [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}}, @@ -34,7 +35,6 @@ var Keys schema.ProgramConfig = schema.ProgramConfig{ "plot_general_colorBackground": true, "plot_general_colorscheme": []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"}, "plot_general_lineWidth": 3, - "plot_list_hideShortRunningJobs": 5 * 60, "plot_list_jobsPerPage": 50, "plot_list_selectedMetrics": []string{"cpu_load", "ipc", "mem_used", "flops_any", "mem_bw"}, "plot_view_plotsPerRow": 3, diff --git a/internal/repository/job.go b/internal/repository/job.go index aa3ff2a..e1e70b9 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -16,6 +16,7 @@ import ( "github.com/99designs/gqlgen/graphql" "github.com/ClusterCockpit/cc-backend/internal/auth" + "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/metricdata" "github.com/ClusterCockpit/cc-backend/pkg/archive" @@ -768,9 +769,6 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { return nil } -// TODO: Move to config -const ShortJobDuration int = 5 * 60 - // GraphQL validation should make sure that no unkown values can be specified. var groupBy2column = map[model.Aggregate]string{ model.AggregateUser: "job.user", @@ -859,7 +857,7 @@ func (r *JobRepository) JobsStatistics(ctx context.Context, } if groupBy == nil { - query := sq.Select("COUNT(job.id)").From("job").Where("job.duration < ?", ShortJobDuration) + query := sq.Select("COUNT(job.id)").From("job").Where("job.duration < ?", config.Keys.ShortRunningJobsDuration) query = SecurityCheck(ctx, query) for _, f := range filter { query = BuildWhereClause(f, query) @@ -870,7 +868,7 @@ func (r *JobRepository) JobsStatistics(ctx context.Context, } } else { col := groupBy2column[*groupBy] - query := sq.Select(col, "COUNT(job.id)").From("job").Where("job.duration < ?", ShortJobDuration) + query := sq.Select(col, "COUNT(job.id)").From("job").Where("job.duration < ?", config.Keys.ShortRunningJobsDuration) query = SecurityCheck(ctx, query) for _, f := range filter { query = BuildWhereClause(f, query) diff --git a/internal/repository/user.go b/internal/repository/user.go index 36d5f2d..2ba1d59 100644 --- a/internal/repository/user.go +++ b/internal/repository/user.go @@ -11,8 +11,8 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/config" - "github.com/ClusterCockpit/cc-backend/pkg/lrucache" "github.com/ClusterCockpit/cc-backend/pkg/log" + "github.com/ClusterCockpit/cc-backend/pkg/lrucache" "github.com/jmoiron/sqlx" ) @@ -63,14 +63,14 @@ func (uCfg *UserCfgRepo) GetUIConfig(user *auth.User) (map[string]interface{}, e } data := uCfg.cache.Get(user.Username, func() (interface{}, time.Duration, int) { - config := make(map[string]interface{}, len(uCfg.uiDefaults)) + uiconfig := make(map[string]interface{}, len(uCfg.uiDefaults)) for k, v := range uCfg.uiDefaults { - config[k] = v + uiconfig[k] = v } rows, err := uCfg.Lookup.Query(user.Username) if err != nil { - log.Warnf("Error while looking up user config for user '%v'", user.Username) + log.Warnf("Error while looking up user uiconfig for user '%v'", user.Username) return err, 0, 0 } @@ -79,22 +79,25 @@ func (uCfg *UserCfgRepo) GetUIConfig(user *auth.User) (map[string]interface{}, e for rows.Next() { var key, rawval string if err := rows.Scan(&key, &rawval); err != nil { - log.Warn("Error while scanning user config values") + log.Warn("Error while scanning user uiconfig values") return err, 0, 0 } var val interface{} if err := json.Unmarshal([]byte(rawval), &val); err != nil { - log.Warn("Error while unmarshaling raw user config json") + log.Warn("Error while unmarshaling raw user uiconfig json") return err, 0, 0 } size += len(key) size += len(rawval) - config[key] = val + uiconfig[key] = val } - return config, 24 * time.Hour, size + // Add global ShortRunningJobsDuration setting as plot_list_hideShortRunningJobs + uiconfig["plot_list_hideShortRunningJobs"] = config.Keys.ShortRunningJobsDuration + + return uiconfig, 24 * time.Hour, size }) if err, ok := data.(error); ok { log.Error("Error in returned dataset") diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 4f2d85c..57138bc 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -14,6 +14,7 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/api" "github.com/ClusterCockpit/cc-backend/internal/auth" + "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/repository" "github.com/ClusterCockpit/cc-backend/pkg/archive" @@ -72,7 +73,7 @@ func setupHomeRoute(i InfoType, r *http.Request) InfoType { from := time.Now().Add(-24 * time.Hour) recentShortJobs, err := jobRepo.CountGroupedJobs(r.Context(), model.AggregateCluster, []*model.JobFilter{{ StartTime: &schema.TimeRange{From: &from, To: nil}, - Duration: &schema.IntRange{From: 0, To: repository.ShortJobDuration}, + Duration: &schema.IntRange{From: 0, To: config.Keys.ShortRunningJobsDuration}, }}, nil, nil) if err != nil { log.Warnf("failed to count jobs: %s", err.Error()) diff --git a/pkg/schema/config.go b/pkg/schema/config.go index 11bd1a9..fa7ba9f 100644 --- a/pkg/schema/config.go +++ b/pkg/schema/config.go @@ -116,6 +116,9 @@ type ProgramConfig struct { // If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. StopJobsExceedingWalltime int `json:"stop-jobs-exceeding-walltime"` + // Defines time X in seconds in which jobs are considered to be "short" and will be filtered in specific views. + ShortRunningJobsDuration int `json:"short-running-jobs-duration"` + // Array of Clusters Clusters []*ClusterConfig `json:"clusters"` } diff --git a/pkg/schema/schemas/config.schema.json b/pkg/schema/schemas/config.schema.json index 1a13e87..c05750c 100644 --- a/pkg/schema/schemas/config.schema.json +++ b/pkg/schema/schemas/config.schema.json @@ -76,6 +76,10 @@ "description": "If not zero, automatically mark jobs as stopped running X seconds longer than their walltime. Only applies if walltime is set for job.", "type": "integer" }, + "short-running-jobs-duration": { + "description": "Do not show running jobs shorter than X seconds.", + "type": "integer" + }, "": { "description": "", "type": "string" @@ -241,10 +245,6 @@ "description": "Jobs shown per page in job lists", "type": "integer" }, - "plot_list_hideShortRunningJobs": { - "description": "Do not show running jobs shorter than X seconds", - "type": "integer" - }, "plot_view_plotsPerRow": { "description": "Number of plots per row in single job view", "type": "integer" @@ -342,8 +342,7 @@ "job_view_polarPlotMetrics", "job_view_selectedMetrics", "plot_general_colorscheme", - "plot_list_selectedMetrics", - "plot_list_hideShortRunningJobs" + "plot_list_selectedMetrics" ] } },