From 9137931b74a252e7192fb50a0978577b7839f5ba Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 15 Jun 2023 11:07:48 +0200 Subject: [PATCH 01/11] Add cpu_load and mem_used to Job stats Fixes #111 --- internal/config/config.go | 4 ++-- internal/importer/handleImport.go | 3 +++ internal/importer/initDB.go | 8 +++++++- internal/repository/job.go | 1 + pkg/schema/schemas/job-data.schema.json | 1 + pkg/schema/schemas/job-meta.schema.json | 1 + 6 files changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index d50fa4f..02a0ba6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,13 +30,13 @@ var Keys schema.ProgramConfig = schema.ProgramConfig{ "analysis_view_histogramMetrics": []string{"flops_any", "mem_bw", "mem_used"}, "analysis_view_scatterPlotMetrics": [][]string{{"flops_any", "mem_bw"}, {"flops_any", "cpu_load"}, {"cpu_load", "mem_bw"}}, "job_view_nodestats_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"}, - "job_view_polarPlotMetrics": []string{"flops_any", "mem_bw", "mem_used", "net_bw", "file_bw"}, + "job_view_polarPlotMetrics": []string{"flops_any", "mem_bw", "mem_used"}, "job_view_selectedMetrics": []string{"flops_any", "mem_bw", "mem_used"}, "plot_general_colorBackground": true, "plot_general_colorscheme": []string{"#00bfff", "#0000ff", "#ff00ff", "#ff0000", "#ff8000", "#ffff00", "#80ff00"}, "plot_general_lineWidth": 3, "plot_list_jobsPerPage": 50, - "plot_list_selectedMetrics": []string{"cpu_load", "ipc", "mem_used", "flops_any", "mem_bw"}, + "plot_list_selectedMetrics": []string{"cpu_load", "mem_used", "flops_any", "mem_bw"}, "plot_view_plotsPerRow": 3, "plot_view_showPolarplot": true, "plot_view_showRoofline": true, diff --git a/internal/importer/handleImport.go b/internal/importer/handleImport.go index 5679d53..bc97ec0 100644 --- a/internal/importer/handleImport.go +++ b/internal/importer/handleImport.go @@ -87,10 +87,13 @@ func HandleImportFlag(flag string) error { } // TODO: Other metrics... + job.LoadAvg = loadJobStat(&jobMeta, "cpu_load") job.FlopsAnyAvg = loadJobStat(&jobMeta, "flops_any") + job.MemUsedMax = loadJobStat(&jobMeta, "mem_used") job.MemBwAvg = loadJobStat(&jobMeta, "mem_bw") job.NetBwAvg = loadJobStat(&jobMeta, "net_bw") job.FileBwAvg = loadJobStat(&jobMeta, "file_bw") + job.RawResources, err = json.Marshal(job.Resources) if err != nil { log.Warn("Error while marshaling job resources") diff --git a/internal/importer/initDB.go b/internal/importer/initDB.go index cce9ebf..3118131 100644 --- a/internal/importer/initDB.go +++ b/internal/importer/initDB.go @@ -61,7 +61,9 @@ func InitDB() error { } // TODO: Other metrics... + job.LoadAvg = loadJobStat(jobMeta, "cpu_load") job.FlopsAnyAvg = loadJobStat(jobMeta, "flops_any") + job.MemUsedMax = loadJobStat(jobMeta, "mem_used") job.MemBwAvg = loadJobStat(jobMeta, "mem_bw") job.NetBwAvg = loadJobStat(jobMeta, "net_bw") job.FileBwAvg = loadJobStat(jobMeta, "file_bw") @@ -150,7 +152,11 @@ func SanityChecks(job *schema.BaseJob) error { func loadJobStat(job *schema.JobMeta, metric string) float64 { if stats, ok := job.Statistics[metric]; ok { - return stats.Avg + if metric == "mem_used" { + return stats.Max + } else { + return stats.Avg + } } return 0.0 diff --git a/internal/repository/job.go b/internal/repository/job.go index 9ae7c1e..0f280cd 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -450,6 +450,7 @@ func (r *JobRepository) MarkArchived( case "mem_bw": stmt = stmt.Set("mem_bw_avg", stats.Avg) case "load": + case "cpu_load": stmt = stmt.Set("load_avg", stats.Avg) case "net_bw": stmt = stmt.Set("net_bw_avg", stats.Avg) diff --git a/pkg/schema/schemas/job-data.schema.json b/pkg/schema/schemas/job-data.schema.json index 1c8d94a..e8a5739 100644 --- a/pkg/schema/schemas/job-data.schema.json +++ b/pkg/schema/schemas/job-data.schema.json @@ -480,6 +480,7 @@ }, "required": [ "cpu_user", + "cpu_load", "mem_used", "flops_any", "mem_bw", diff --git a/pkg/schema/schemas/job-meta.schema.json b/pkg/schema/schemas/job-meta.schema.json index 1e9ad5c..b907d7f 100644 --- a/pkg/schema/schemas/job-meta.schema.json +++ b/pkg/schema/schemas/job-meta.schema.json @@ -327,6 +327,7 @@ }, "required": [ "cpu_user", + "cpu_load", "mem_used", "flops_any", "mem_bw" From 17113eddca71c271dc20440838439161d4500130 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 15 Jun 2023 11:23:50 +0200 Subject: [PATCH 02/11] Fix failed test due to config change --- internal/repository/user_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/repository/user_test.go b/internal/repository/user_test.go index 084441e..ac3b0d5 100644 --- a/internal/repository/user_test.go +++ b/internal/repository/user_test.go @@ -63,7 +63,7 @@ func TestGetUIConfig(t *testing.T) { tmp := cfg["plot_list_selectedMetrics"] metrics := tmp.([]string) str := metrics[2] - if str != "mem_used" { - t.Errorf("wrong config\ngot: %s \nwant: mem_bw", str) + if str != "flops_any" { + t.Errorf("wrong config\ngot: %s \nwant: flops_any", str) } } From a9544f5609d89fe3b3723cf55a8d42f646f7a417 Mon Sep 17 00:00:00 2001 From: Pay Giesselmann Date: Tue, 20 Jun 2023 15:47:38 +0200 Subject: [PATCH 03/11] lower log level for frequent messages --- cmd/cc-backend/main.go | 2 +- internal/api/rest.go | 2 +- internal/auth/auth.go | 6 +++--- internal/repository/hooks.go | 4 ++-- internal/repository/job.go | 26 +++++++++++++------------- internal/repository/query.go | 2 +- internal/repository/stats.go | 14 +++++++------- web/web.go | 2 +- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index f9868e6..ae2e044 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -338,7 +338,7 @@ func main() { handlers.AllowedOrigins([]string{"*"}))) handler := handlers.CustomLoggingHandler(io.Discard, r, func(_ io.Writer, params handlers.LogFormatterParams) { if strings.HasPrefix(params.Request.RequestURI, "/api/") { - log.Infof("%s %s (%d, %.02fkb, %dms)", + log.Debugf("%s %s (%d, %.02fkb, %dms)", params.Request.Method, params.URL.RequestURI(), params.StatusCode, float32(params.Size)/1024, time.Since(params.TimeStamp).Milliseconds()) diff --git a/internal/api/rest.go b/internal/api/rest.go index da37a25..c199bc2 100644 --- a/internal/api/rest.go +++ b/internal/api/rest.go @@ -523,7 +523,7 @@ func (api *RestApi) startJob(rw http.ResponseWriter, r *http.Request) { } else if err == nil { for _, job := range jobs { if (req.StartTime - job.StartTimeUnix) < 86400 { - handleError(fmt.Errorf("a job with that jobId, cluster and startTime already exists: dbid: %d", job.ID), http.StatusUnprocessableEntity, rw) + handleError(fmt.Errorf("a job with that jobId, cluster and startTime already exists: dbid: %d, jobid: %d", job.ID, job.JobID), http.StatusUnprocessableEntity, rw) return } } diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 9234e9a..ea6c9f2 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -364,7 +364,7 @@ func (auth *Authentication) Login( return } - log.Warn("login failed: no authenticator applied") + log.Debugf("login failed: no authenticator applied") onfailure(rw, r, err) }) } @@ -380,7 +380,7 @@ func (auth *Authentication) Auth( for _, authenticator := range auth.authenticators { user, err := authenticator.Auth(rw, r) if err != nil { - log.Warnf("authentication failed: %s", err.Error()) + log.Infof("authentication failed: %s", err.Error()) http.Error(rw, err.Error(), http.StatusUnauthorized) return } @@ -393,7 +393,7 @@ func (auth *Authentication) Auth( return } - log.Warnf("authentication failed: %s", "no authenticator applied") + log.Debugf("authentication failed: %s", "no authenticator applied") // http.Error(rw, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) onfailure(rw, r, errors.New("unauthorized (login first or use a token)")) }) diff --git a/internal/repository/hooks.go b/internal/repository/hooks.go index 57fac8a..2f40fd5 100644 --- a/internal/repository/hooks.go +++ b/internal/repository/hooks.go @@ -16,13 +16,13 @@ type Hooks struct{} // Before hook will print the query with it's args and return the context with the timestamp func (h *Hooks) Before(ctx context.Context, query string, args ...interface{}) (context.Context, error) { - log.Infof("SQL query %s %q", query, args) + log.Debugf("SQL query %s %q", query, args) return context.WithValue(ctx, "begin", time.Now()), nil } // After hook will get the timestamp registered on the Before hook and print the elapsed time func (h *Hooks) After(ctx context.Context, query string, args ...interface{}) (context.Context, error) { begin := ctx.Value("begin").(time.Time) - log.Infof("Took: %s\n", time.Since(begin)) + log.Debugf("Took: %s\n", time.Since(begin)) return ctx, nil } diff --git a/internal/repository/job.go b/internal/repository/job.go index 0f280cd..ff65989 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -55,7 +55,6 @@ func GetJobRepository() *JobRepository { // start archiving worker go jobRepoInstance.archivingWorker() }) - return jobRepoInstance } @@ -178,7 +177,7 @@ func (r *JobRepository) FetchMetadata(job *schema.Job) (map[string]string, error } r.cache.Put(cachekey, job.MetaData, len(job.RawMetaData), 24*time.Hour) - log.Infof("Timer FetchMetadata %s", time.Since(start)) + log.Debugf("Timer FetchMetadata %s", time.Since(start)) return job.MetaData, nil } @@ -238,7 +237,7 @@ func (r *JobRepository) Find( q = q.Where("job.start_time = ?", *startTime) } - log.Infof("Timer Find %s", time.Since(start)) + log.Debugf("Timer Find %s", time.Since(start)) return scanJob(q.RunWith(r.stmtCache).QueryRow()) } @@ -278,7 +277,7 @@ func (r *JobRepository) FindAll( } jobs = append(jobs, job) } - log.Infof("Timer FindAll %s", time.Since(start)) + log.Debugf("Timer FindAll %s", time.Since(start)) return jobs, nil } @@ -344,7 +343,7 @@ func (r *JobRepository) DeleteJobsBefore(startTime int64) (int, error) { if err != nil { log.Errorf(" DeleteJobsBefore(%d): error %#v", startTime, err) } else { - log.Infof("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt) + log.Debugf("DeleteJobsBefore(%d): Deleted %d jobs", startTime, cnt) } return cnt, err } @@ -354,7 +353,7 @@ func (r *JobRepository) DeleteJobById(id int64) error { if err != nil { log.Errorf("DeleteJobById(%d): error %#v", id, err) } else { - log.Infof("DeleteJobById(%d): Success", id) + log.Debugf("DeleteJobById(%d): Success", id) } return err } @@ -383,7 +382,7 @@ func (r *JobRepository) CountGroupedJobs( count = fmt.Sprintf(`sum(job.num_nodes * (CASE WHEN job.job_state = "running" THEN %d - job.start_time ELSE job.duration END)) as count`, now) runner = r.DB default: - log.Infof("CountGroupedJobs() Weight %v unknown.", *weight) + log.Debugf("CountGroupedJobs() Weight %v unknown.", *weight) } } @@ -418,7 +417,7 @@ func (r *JobRepository) CountGroupedJobs( counts[group] = count } - log.Infof("Timer CountGroupedJobs %s", time.Since(start)) + log.Debugf("Timer CountGroupedJobs %s", time.Since(start)) return counts, nil } @@ -457,7 +456,7 @@ func (r *JobRepository) MarkArchived( case "file_bw": stmt = stmt.Set("file_bw_avg", stats.Avg) default: - log.Infof("MarkArchived() Metric '%v' unknown", metric) + log.Debugf("MarkArchived() Metric '%v' unknown", metric) } } @@ -476,6 +475,7 @@ func (r *JobRepository) archivingWorker() { if !ok { break } + start := time.Now() // not using meta data, called to load JobMeta into Cache? // will fail if job meta not in repository if _, err := r.FetchMetadata(job); err != nil { @@ -498,7 +498,7 @@ func (r *JobRepository) archivingWorker() { log.Errorf("archiving job (dbid: %d) failed: %s", job.ID, err.Error()) continue } - + log.Debugf("archiving job %d took %s", job.JobID, time.Since(start)) log.Printf("archiving job (dbid: %d) successful", job.ID) r.archivePending.Done() } @@ -635,7 +635,7 @@ func (r *JobRepository) Partitions(cluster string) ([]string, error) { if err != nil { return nil, err } - log.Infof("Timer Partitions %s", time.Since(start)) + log.Debugf("Timer Partitions %s", time.Since(start)) return partitions.([]string), nil } @@ -680,7 +680,7 @@ func (r *JobRepository) AllocatedNodes(cluster string) (map[string]map[string]in } } - log.Infof("Timer AllocatedNodes %s", time.Since(start)) + log.Debugf("Timer AllocatedNodes %s", time.Since(start)) return subclusters, nil } @@ -709,7 +709,7 @@ func (r *JobRepository) StopJobsExceedingWalltimeBy(seconds int) error { if rowsAffected > 0 { log.Infof("%d jobs have been marked as failed due to running too long", rowsAffected) } - log.Infof("Timer StopJobsExceedingWalltimeBy %s", time.Since(start)) + log.Debugf("Timer StopJobsExceedingWalltimeBy %s", time.Since(start)) return nil } diff --git a/internal/repository/query.go b/internal/repository/query.go index ba1d1d7..56ba173 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -211,7 +211,7 @@ func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilde if len(user.Projects) != 0 { return query.Where(sq.Or{sq.Eq{"job.project": user.Projects}, sq.Eq{"job.user": user.Username}}), nil } else { - log.Infof("Manager-User '%s' has no defined projects to lookup! Query only personal jobs ...", user.Username) + log.Debugf("Manager-User '%s' has no defined projects to lookup! Query only personal jobs ...", user.Username) return query.Where("job.user = ?", user.Username), nil } } else if user.HasRole(auth.RoleUser) { // User : Only personal jobs diff --git a/internal/repository/stats.go b/internal/repository/stats.go index d96c20a..158d34a 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -171,7 +171,7 @@ func (r *JobRepository) JobsStatsGrouped( } } - log.Infof("Timer JobsStatsGrouped %s", time.Since(start)) + log.Debugf("Timer JobsStatsGrouped %s", time.Since(start)) return stats, nil } @@ -212,7 +212,7 @@ func (r *JobRepository) JobsStats( TotalAccHours: totalAccHours}) } - log.Infof("Timer JobStats %s", time.Since(start)) + log.Debugf("Timer JobStats %s", time.Since(start)) return stats, nil } @@ -251,7 +251,7 @@ func (r *JobRepository) JobCountGrouped( } } - log.Infof("Timer JobCountGrouped %s", time.Since(start)) + log.Debugf("Timer JobCountGrouped %s", time.Since(start)) return stats, nil } @@ -300,7 +300,7 @@ func (r *JobRepository) AddJobCountGrouped( } } - log.Infof("Timer AddJobCountGrouped %s", time.Since(start)) + log.Debugf("Timer AddJobCountGrouped %s", time.Since(start)) return stats, nil } @@ -343,7 +343,7 @@ func (r *JobRepository) AddJobCount( } } - log.Infof("Timer JobJobCount %s", time.Since(start)) + log.Debugf("Timer JobJobCount %s", time.Since(start)) return stats, nil } @@ -368,7 +368,7 @@ func (r *JobRepository) AddHistograms( return nil, err } - log.Infof("Timer AddHistograms %s", time.Since(start)) + log.Debugf("Timer AddHistograms %s", time.Since(start)) return stat, nil } @@ -406,6 +406,6 @@ func (r *JobRepository) jobsStatisticsHistogram( points = append(points, &point) } - log.Infof("Timer jobsStatisticsHistogram %s", time.Since(start)) + log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start)) return points, nil } diff --git a/web/web.go b/web/web.go index 3547b85..6f31333 100644 --- a/web/web.go +++ b/web/web.go @@ -106,7 +106,7 @@ func RenderTemplate(rw http.ResponseWriter, r *http.Request, file string, page * } } - log.Infof("Page config : %v\n", page.Config) + log.Debugf("Page config : %v\n", page.Config) if err := t.Execute(rw, page); err != nil { log.Errorf("Template error: %s", err.Error()) } From bcadb1addaab819bf7c41e5932f8df7673b1c4aa Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 22 Jun 2023 10:58:36 +0200 Subject: [PATCH 04/11] Remove errorcase from single searchterm logic --- internal/repository/job.go | 3 +-- internal/routerConfig/routes.go | 40 +++++++++++++++++---------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/internal/repository/job.go b/internal/repository/job.go index 9ae7c1e..728d1c0 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -523,11 +523,10 @@ var ErrForbidden = errors.New("not authorized") // If query is found to be an integer (= conversion to INT datatype succeeds), skip back to parent call // If nothing matches the search, `ErrNotFound` is returned. -func (r *JobRepository) FindUserOrProjectOrJobname(ctx context.Context, searchterm string) (username string, project string, metasnip string, err error) { +func (r *JobRepository) FindUserOrProjectOrJobname(user *auth.User, searchterm string) (username string, project string, metasnip string, err error) { if _, err := strconv.Atoi(searchterm); err == nil { // Return empty on successful conversion: parent method will redirect for integer jobId return "", "", "", nil } else { // Has to have letters and logged-in user for other guesses - user := auth.GetUser(ctx) if user != nil { // Find username in jobs (match) uresult, _ := r.FindColumnValue(user, searchterm, "job", "user", "user", false) diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 2aa3f05..94d8259 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -278,55 +278,57 @@ func HandleSearchBar(rw http.ResponseWriter, r *http.Request, api *api.RestApi) if len(splitSearch) == 2 { switch strings.Trim(splitSearch[0], " ") { case "jobId": - http.Redirect(rw, r, "/monitoring/jobs/?jobId="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusTemporaryRedirect) // All Users: Redirect to Tablequery + http.Redirect(rw, r, "/monitoring/jobs/?jobId="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusFound) // All Users: Redirect to Tablequery case "jobName": - http.Redirect(rw, r, "/monitoring/jobs/?jobName="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusTemporaryRedirect) // All Users: Redirect to Tablequery + http.Redirect(rw, r, "/monitoring/jobs/?jobName="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusFound) // All Users: Redirect to Tablequery case "projectId": - http.Redirect(rw, r, "/monitoring/jobs/?projectMatch=eq&project="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusTemporaryRedirect) // All Users: Redirect to Tablequery + http.Redirect(rw, r, "/monitoring/jobs/?projectMatch=eq&project="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusFound) // All Users: Redirect to Tablequery case "username": if user.HasAnyRole([]auth.Role{auth.RoleAdmin, auth.RoleSupport, auth.RoleManager}) { - http.Redirect(rw, r, "/monitoring/users/?user="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusTemporaryRedirect) + http.Redirect(rw, r, "/monitoring/users/?user="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusFound) } else { - http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusTemporaryRedirect) // Users: Redirect to Tablequery + http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusPermanentRedirect) // Users: Redirect to Tablequery } case "name": usernames, _ := api.JobRepository.FindColumnValues(user, strings.Trim(splitSearch[1], " "), "user", "username", "name") if len(usernames) != 0 { joinedNames := strings.Join(usernames, "&user=") - http.Redirect(rw, r, "/monitoring/users/?user="+joinedNames, http.StatusTemporaryRedirect) + http.Redirect(rw, r, "/monitoring/users/?user="+joinedNames, http.StatusFound) } else { if user.HasAnyRole([]auth.Role{auth.RoleAdmin, auth.RoleSupport, auth.RoleManager}) { - http.Redirect(rw, r, "/monitoring/users/?user=NoUserNameFound", http.StatusTemporaryRedirect) + http.Redirect(rw, r, "/monitoring/users/?user=NoUserNameFound", http.StatusPermanentRedirect) } else { - http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusTemporaryRedirect) // Users: Redirect to Tablequery + http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusPermanentRedirect) // Users: Redirect to Tablequery } } default: log.Warnf("Searchbar type parameter '%s' unknown", strings.Trim(splitSearch[0], " ")) - http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusTemporaryRedirect) // Unknown: Redirect to Tablequery + http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusPermanentRedirect) // Unknown: Redirect to Tablequery } } else if len(splitSearch) == 1 { - username, project, jobname, err := api.JobRepository.FindUserOrProjectOrJobname(r.Context(), strings.Trim(search, " ")) - if err != nil { - log.Errorf("Error while searchbar best guess: %v", err.Error()) - http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusTemporaryRedirect) // Unknown: Redirect to Tablequery - } + username, project, jobname, _ := api.JobRepository.FindUserOrProjectOrJobname(user, strings.Trim(search, " ")) + + /* Causes 'http: superfluous response.WriteHeader call' causing SSL error and frontend crash: Cause unknown*/ + // if err != nil { + // log.Errorf("Error while searchbar best guess: %v", err.Error()) + // http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusPermanentRedirect) // Unknown: Redirect to Tablequery + //} if username != "" { - http.Redirect(rw, r, "/monitoring/user/"+username, http.StatusTemporaryRedirect) // User: Redirect to user page + http.Redirect(rw, r, "/monitoring/user/"+username, http.StatusFound) // User: Redirect to user page } else if project != "" { - http.Redirect(rw, r, "/monitoring/jobs/?projectMatch=eq&project="+url.QueryEscape(strings.Trim(search, " ")), http.StatusTemporaryRedirect) // projectId (equal) + http.Redirect(rw, r, "/monitoring/jobs/?projectMatch=eq&project="+url.QueryEscape(strings.Trim(search, " ")), http.StatusFound) // projectId (equal) } else if jobname != "" { - http.Redirect(rw, r, "/monitoring/jobs/?jobName="+url.QueryEscape(strings.Trim(search, " ")), http.StatusTemporaryRedirect) // JobName (contains) + http.Redirect(rw, r, "/monitoring/jobs/?jobName="+url.QueryEscape(strings.Trim(search, " ")), http.StatusFound) // JobName (contains) } else { - http.Redirect(rw, r, "/monitoring/jobs/?jobId="+url.QueryEscape(strings.Trim(search, " ")), http.StatusTemporaryRedirect) // No Result: Probably jobId + http.Redirect(rw, r, "/monitoring/jobs/?jobId="+url.QueryEscape(strings.Trim(search, " ")), http.StatusFound) // No Result: Probably jobId } } else { log.Warnf("Searchbar query parameters malformed: %v", search) - http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusTemporaryRedirect) // Unknown: Redirect to Tablequery + http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusPermanentRedirect) // Unknown: Redirect to Tablequery } } else { From 82b8e8c284a573593186129f9fa38dd8b97edad7 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 22 Jun 2023 16:26:09 +0200 Subject: [PATCH 05/11] Fix bug in SearchBar Handler Introduce Message boxes Incomplete and needs cleanup --- cmd/cc-backend/main.go | 2 +- internal/routerConfig/routes.go | 37 +++++++++++++++------------ web/templates/message.tmpl | 21 +++++++++++++++ web/web.go | 45 ++++++++++++++++++++++++++++++--- 4 files changed, 85 insertions(+), 20 deletions(-) create mode 100644 web/templates/message.tmpl diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index f9868e6..164d7f1 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -316,7 +316,7 @@ func main() { // Send a searchId and then reply with a redirect to a user, or directly send query to job table for jobid and project. secured.HandleFunc("/search", func(rw http.ResponseWriter, r *http.Request) { - routerConfig.HandleSearchBar(rw, r, api) + routerConfig.HandleSearchBar(rw, r) }) // Mount all /monitoring/... and /api/... routes. diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 94d8259..9000c18 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -12,7 +12,6 @@ import ( "strings" "time" - "github.com/ClusterCockpit/cc-backend/internal/api" "github.com/ClusterCockpit/cc-backend/internal/auth" "github.com/ClusterCockpit/cc-backend/internal/graph/model" "github.com/ClusterCockpit/cc-backend/internal/repository" @@ -270,8 +269,9 @@ func SetupRoutes(router *mux.Router, version string, hash string, buildTime stri } } -func HandleSearchBar(rw http.ResponseWriter, r *http.Request, api *api.RestApi) { +func HandleSearchBar(rw http.ResponseWriter, r *http.Request) { if search := r.URL.Query().Get("searchId"); search != "" { + repo := repository.GetJobRepository() user := auth.GetUser(r.Context()) splitSearch := strings.Split(search, ":") @@ -287,10 +287,11 @@ func HandleSearchBar(rw http.ResponseWriter, r *http.Request, api *api.RestApi) if user.HasAnyRole([]auth.Role{auth.RoleAdmin, auth.RoleSupport, auth.RoleManager}) { http.Redirect(rw, r, "/monitoring/users/?user="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusFound) } else { - http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusPermanentRedirect) // Users: Redirect to Tablequery + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: "Missing Access Rights"}) + // web.RenderMessage(rw, "error", "Missing access rights!") } case "name": - usernames, _ := api.JobRepository.FindColumnValues(user, strings.Trim(splitSearch[1], " "), "user", "username", "name") + usernames, _ := repo.FindColumnValues(user, strings.Trim(splitSearch[1], " "), "user", "username", "name") if len(usernames) != 0 { joinedNames := strings.Join(usernames, "&user=") http.Redirect(rw, r, "/monitoring/users/?user="+joinedNames, http.StatusFound) @@ -298,23 +299,27 @@ func HandleSearchBar(rw http.ResponseWriter, r *http.Request, api *api.RestApi) if user.HasAnyRole([]auth.Role{auth.RoleAdmin, auth.RoleSupport, auth.RoleManager}) { http.Redirect(rw, r, "/monitoring/users/?user=NoUserNameFound", http.StatusPermanentRedirect) } else { - http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusPermanentRedirect) // Users: Redirect to Tablequery + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: "Missing Access Rights"}) + // web.RenderMessage(rw, "error", "Missing access rights!") } } default: - log.Warnf("Searchbar type parameter '%s' unknown", strings.Trim(splitSearch[0], " ")) - http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusPermanentRedirect) // Unknown: Redirect to Tablequery + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: fmt.Sprintf("Unknown search term %s", strings.Trim(splitSearch[0], " "))}) + // web.RenderMessage(rw, "error", fmt.Sprintf("Unknown search term %s", strings.Trim(splitSearch[0], " "))) } } else if len(splitSearch) == 1 { - username, project, jobname, _ := api.JobRepository.FindUserOrProjectOrJobname(user, strings.Trim(search, " ")) + username, project, jobname, err := repo.FindUserOrProjectOrJobname(user, strings.Trim(search, " ")) + // err := fmt.Errorf("Blabla") /* Causes 'http: superfluous response.WriteHeader call' causing SSL error and frontend crash: Cause unknown*/ - // if err != nil { - // log.Errorf("Error while searchbar best guess: %v", err.Error()) - // http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusPermanentRedirect) // Unknown: Redirect to Tablequery - //} + if err != nil { + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: "No search result"}) + return + // web.RenderMessage(rw, "info", "Search with no result") + // log.Errorf("Error while searchbar best guess: %v", err.Error()) + } if username != "" { http.Redirect(rw, r, "/monitoring/user/"+username, http.StatusFound) // User: Redirect to user page @@ -327,11 +332,11 @@ func HandleSearchBar(rw http.ResponseWriter, r *http.Request, api *api.RestApi) } } else { - log.Warnf("Searchbar query parameters malformed: %v", search) - http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusPermanentRedirect) // Unknown: Redirect to Tablequery + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: "Searchbar query parameters malformed"}) + // web.RenderMessage(rw, "warn", "Searchbar query parameters malformed") } - } else { - http.Redirect(rw, r, "/monitoring/jobs/?", http.StatusTemporaryRedirect) + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: "Empty search"}) + // web.RenderMessage(rw, "warn", "Empty search") } } diff --git a/web/templates/message.tmpl b/web/templates/message.tmpl new file mode 100644 index 0000000..72e5d69 --- /dev/null +++ b/web/templates/message.tmpl @@ -0,0 +1,21 @@ +{{define "navigation"}} +
+ +
+{{end}} + +{{define "content"}} +
+
+ +
+
+{{end}} diff --git a/web/web.go b/web/web.go index 3547b85..04293d1 100644 --- a/web/web.go +++ b/web/web.go @@ -96,8 +96,7 @@ type Page struct { func RenderTemplate(rw http.ResponseWriter, r *http.Request, file string, page *Page) { t, ok := templates[file] if !ok { - log.Fatalf("WEB/WEB > template '%s' not found", file) - panic("template not found") + log.Errorf("WEB/WEB > template '%s' not found", file) } if page.Clusters == nil { @@ -106,7 +105,47 @@ func RenderTemplate(rw http.ResponseWriter, r *http.Request, file string, page * } } - log.Infof("Page config : %v\n", page.Config) + log.Debugf("Page config : %v\n", page.Config) + if err := t.Execute(rw, page); err != nil { + log.Errorf("Template error: %s", err.Error()) + } +} + +type Message struct { + Title string + Type string + Message string + Icon string +} + +func RenderMessage(rw http.ResponseWriter, msgType string, msg string) { + var page Message + log.Info("render message template") + + switch msgType { + case "success": + page.Title = "Success" + page.Type = "alert-success" + case "info": + page.Title = "Info" + page.Type = "alert-info" + case "warn": + page.Title = "Warning" + page.Type = "alert-warning" + case "error": + page.Title = "Error" + page.Type = "alert-danger" + default: + page.Title = "Message" + page.Type = "alert-secondary" + } + t, ok := templates["message.tmpl"] + if !ok { + log.Error("WEB/WEB > template message.tmpl not found") + } + page.Message = msg + rw.Header().Add("Content-Type", "text/html; charset=utf-8") + if err := t.Execute(rw, page); err != nil { log.Errorf("Template error: %s", err.Error()) } From 8eda4b306d0c56dce2a76ef846a569e8b0de7323 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 22 Jun 2023 18:09:40 +0200 Subject: [PATCH 06/11] Unify and cleanup message template --- cmd/cc-backend/main.go | 21 +++++++++------- internal/routerConfig/routes.go | 24 +++++------------- web/templates/login.tmpl | 12 +++------ web/templates/message.tmpl | 4 +-- web/web.go | 44 ++------------------------------- 5 files changed, 25 insertions(+), 80 deletions(-) diff --git a/cmd/cc-backend/main.go b/cmd/cc-backend/main.go index 164d7f1..b591246 100644 --- a/cmd/cc-backend/main.go +++ b/cmd/cc-backend/main.go @@ -274,9 +274,10 @@ func main() { rw.Header().Add("Content-Type", "text/html; charset=utf-8") rw.WriteHeader(http.StatusUnauthorized) web.RenderTemplate(rw, r, "login.tmpl", &web.Page{ - Title: "Login failed - ClusterCockpit", - Error: err.Error(), - Build: buildInfo, + Title: "Login failed - ClusterCockpit", + MsgType: "alert-warning", + Message: err.Error(), + Build: buildInfo, }) })).Methods(http.MethodPost) @@ -284,9 +285,10 @@ func main() { rw.Header().Add("Content-Type", "text/html; charset=utf-8") rw.WriteHeader(http.StatusOK) web.RenderTemplate(rw, r, "login.tmpl", &web.Page{ - Title: "Bye - ClusterCockpit", - Info: "Logout sucessful", - Build: buildInfo, + Title: "Bye - ClusterCockpit", + MsgType: "alert-info", + Message: "Logout successful", + Build: buildInfo, }) }))).Methods(http.MethodPost) @@ -299,9 +301,10 @@ func main() { func(rw http.ResponseWriter, r *http.Request, err error) { rw.WriteHeader(http.StatusUnauthorized) web.RenderTemplate(rw, r, "login.tmpl", &web.Page{ - Title: "Authentication failed - ClusterCockpit", - Error: err.Error(), - Build: buildInfo, + Title: "Authentication failed - ClusterCockpit", + MsgType: "alert-danger", + Message: err.Error(), + Build: buildInfo, }) }) }) diff --git a/internal/routerConfig/routes.go b/internal/routerConfig/routes.go index 9000c18..45f77c5 100644 --- a/internal/routerConfig/routes.go +++ b/internal/routerConfig/routes.go @@ -287,8 +287,7 @@ func HandleSearchBar(rw http.ResponseWriter, r *http.Request) { if user.HasAnyRole([]auth.Role{auth.RoleAdmin, auth.RoleSupport, auth.RoleManager}) { http.Redirect(rw, r, "/monitoring/users/?user="+url.QueryEscape(strings.Trim(splitSearch[1], " ")), http.StatusFound) } else { - web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: "Missing Access Rights"}) - // web.RenderMessage(rw, "error", "Missing access rights!") + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Error", MsgType: "alert-danger", Message: "Missing Access Rights"}) } case "name": usernames, _ := repo.FindColumnValues(user, strings.Trim(splitSearch[1], " "), "user", "username", "name") @@ -299,26 +298,18 @@ func HandleSearchBar(rw http.ResponseWriter, r *http.Request) { if user.HasAnyRole([]auth.Role{auth.RoleAdmin, auth.RoleSupport, auth.RoleManager}) { http.Redirect(rw, r, "/monitoring/users/?user=NoUserNameFound", http.StatusPermanentRedirect) } else { - web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: "Missing Access Rights"}) - // web.RenderMessage(rw, "error", "Missing access rights!") + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Error", MsgType: "alert-danger", Message: "Missing Access Rights"}) } } default: - web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: fmt.Sprintf("Unknown search term %s", strings.Trim(splitSearch[0], " "))}) - // web.RenderMessage(rw, "error", fmt.Sprintf("Unknown search term %s", strings.Trim(splitSearch[0], " "))) + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warning", MsgType: "alert-warning", Message: fmt.Sprintf("Unknown search term %s", strings.Trim(splitSearch[0], " "))}) } - } else if len(splitSearch) == 1 { username, project, jobname, err := repo.FindUserOrProjectOrJobname(user, strings.Trim(search, " ")) - // err := fmt.Errorf("Blabla") - - /* Causes 'http: superfluous response.WriteHeader call' causing SSL error and frontend crash: Cause unknown*/ if err != nil { - web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: "No search result"}) + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Info", MsgType: "alert-info", Message: "Search without result"}) return - // web.RenderMessage(rw, "info", "Search with no result") - // log.Errorf("Error while searchbar best guess: %v", err.Error()) } if username != "" { @@ -330,13 +321,10 @@ func HandleSearchBar(rw http.ResponseWriter, r *http.Request) { } else { http.Redirect(rw, r, "/monitoring/jobs/?jobId="+url.QueryEscape(strings.Trim(search, " ")), http.StatusFound) // No Result: Probably jobId } - } else { - web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: "Searchbar query parameters malformed"}) - // web.RenderMessage(rw, "warn", "Searchbar query parameters malformed") + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Error", MsgType: "alert-danger", Message: "Searchbar query parameters malformed"}) } } else { - web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warn", Info: "Empty search"}) - // web.RenderMessage(rw, "warn", "Empty search") + web.RenderTemplate(rw, r, "message.tmpl", &web.Page{Title: "Warning", MsgType: "alert-warning", Message: "Empty search"}) } } diff --git a/web/templates/login.tmpl b/web/templates/login.tmpl index 1e689b8..304a96f 100644 --- a/web/templates/login.tmpl +++ b/web/templates/login.tmpl @@ -17,15 +17,9 @@
- {{if .Error}} - - {{end}} - - {{if .Info}} -